1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4""" 5These transformations take a task description and turn it into a TaskCluster 6task definition (along with attributes, label, etc.). The input to these 7transformations is generic to any kind of task, but abstracts away some of the 8complexities of worker implementations, scopes, and treeherder annotations. 9""" 10 11from __future__ import absolute_import, print_function, unicode_literals 12 13import hashlib 14import os 15import re 16import time 17from copy import deepcopy 18import six 19from six import text_type 20 21import attr 22 23from mozbuild.util import memoize 24from taskgraph.util.attributes import TRUNK_PROJECTS 25from taskgraph.util.hash import hash_path 26from taskgraph.util.treeherder import split_symbol 27from taskgraph.transforms.base import TransformSequence 28from taskgraph.util.keyed_by import evaluate_keyed_by 29from taskgraph.util.schema import ( 30 validate_schema, 31 Schema, 32 optionally_keyed_by, 33 resolve_keyed_by, 34 taskref_or_string, 35) 36from taskgraph.optimize.schema import OptimizationSchema 37from taskgraph.util.partners import get_partners_to_be_published 38from taskgraph.util.scriptworker import ( 39 BALROG_ACTIONS, 40 get_release_config, 41) 42from taskgraph.util.signed_artifacts import get_signed_artifacts 43from taskgraph.util.time import value_of 44from taskgraph.util.workertypes import worker_type_implementation 45from voluptuous import Any, Required, Optional, Extra, Match, All, NotIn 46from taskgraph import GECKO, MAX_DEPENDENCIES 47from ..util import docker as dockerutil 48from ..util.workertypes import get_worker_type 49 50RUN_TASK = os.path.join(GECKO, "taskcluster", "scripts", "run-task") 51 52SCCACHE_GCS_PROJECT = "sccache-3" 53 54 55@memoize 56def _run_task_suffix(): 57 """String to append to cache names under control of run-task.""" 58 return hash_path(RUN_TASK)[0:20] 59 60 61def _compute_geckoview_version(app_version, moz_build_date): 62 """Geckoview version string that matches geckoview gradle configuration""" 63 # Must be synchronized with /mobile/android/geckoview/build.gradle computeVersionCode(...) 64 version_without_milestone = re.sub(r"a[0-9]", "", app_version, 1) 65 parts = version_without_milestone.split(".") 66 return "%s.%s.%s" % (parts[0], parts[1], moz_build_date) 67 68 69# A task description is a general description of a TaskCluster task 70task_description_schema = Schema( 71 { 72 # the label for this task 73 Required("label"): text_type, 74 # description of the task (for metadata) 75 Required("description"): text_type, 76 # attributes for this task 77 Optional("attributes"): {text_type: object}, 78 # relative path (from config.path) to the file task was defined in 79 Optional("job-from"): text_type, 80 # dependencies of this task, keyed by name; these are passed through 81 # verbatim and subject to the interpretation of the Task's get_dependencies 82 # method. 83 Optional("dependencies"): { 84 All( 85 text_type, 86 NotIn( 87 ["self", "decision"], 88 "Can't use 'self` or 'decision' as depdency names.", 89 ), 90 ): object, 91 }, 92 # Soft dependencies of this task, as a list of tasks labels 93 Optional("soft-dependencies"): [text_type], 94 # Dependencies that must be scheduled in order for this task to run. 95 Optional("if-dependencies"): [text_type], 96 Optional("requires"): Any("all-completed", "all-resolved"), 97 # expiration and deadline times, relative to task creation, with units 98 # (e.g., "14 days"). Defaults are set based on the project. 99 Optional("expires-after"): text_type, 100 Optional("deadline-after"): text_type, 101 # custom routes for this task; the default treeherder routes will be added 102 # automatically 103 Optional("routes"): [text_type], 104 # custom scopes for this task; any scopes required for the worker will be 105 # added automatically. The following parameters will be substituted in each 106 # scope: 107 # {level} -- the scm level of this push 108 # {project} -- the project of this push 109 Optional("scopes"): [text_type], 110 # Tags 111 Optional("tags"): {text_type: text_type}, 112 # custom "task.extra" content 113 Optional("extra"): {text_type: object}, 114 # treeherder-related information; see 115 # https://firefox-ci-tc.services.mozilla.com/schemas/taskcluster-treeherder/v1/task-treeherder-config.json 116 # If not specified, no treeherder extra information or routes will be 117 # added to the task 118 Optional("treeherder"): { 119 # either a bare symbol, or "grp(sym)". 120 "symbol": text_type, 121 # the job kind 122 "kind": Any("build", "test", "other"), 123 # tier for this task 124 "tier": int, 125 # task platform, in the form platform/collection, used to set 126 # treeherder.machine.platform and treeherder.collection or 127 # treeherder.labels 128 "platform": Match("^[A-Za-z0-9_-]{1,50}/[A-Za-z0-9_-]{1,50}$"), 129 }, 130 # information for indexing this build so its artifacts can be discovered; 131 # if omitted, the build will not be indexed. 132 Optional("index"): { 133 # the name of the product this build produces 134 "product": text_type, 135 # the names to use for this job in the TaskCluster index 136 "job-name": text_type, 137 # Type of gecko v2 index to use 138 "type": Any( 139 "generic", 140 "l10n", 141 "shippable", 142 "shippable-l10n", 143 "android-shippable", 144 "android-shippable-with-multi-l10n", 145 "shippable-with-multi-l10n", 146 ), 147 # The rank that the task will receive in the TaskCluster 148 # index. A newly completed task supercedes the currently 149 # indexed task iff it has a higher rank. If unspecified, 150 # 'by-tier' behavior will be used. 151 "rank": Any( 152 # Rank is equal the timestamp of the build_date for tier-1 153 # tasks, and zero for non-tier-1. This sorts tier-{2,3} 154 # builds below tier-1 in the index. 155 "by-tier", 156 # Rank is given as an integer constant (e.g. zero to make 157 # sure a task is last in the index). 158 int, 159 # Rank is equal to the timestamp of the build_date. This 160 # option can be used to override the 'by-tier' behavior 161 # for non-tier-1 tasks. 162 "build_date", 163 ), 164 }, 165 # The `run_on_projects` attribute, defaulting to "all". This dictates the 166 # projects on which this task should be included in the target task set. 167 # See the attributes documentation for details. 168 Optional("run-on-projects"): optionally_keyed_by("build-platform", [text_type]), 169 # Like `run_on_projects`, `run-on-hg-branches` defaults to "all". 170 Optional("run-on-hg-branches"): optionally_keyed_by("project", [text_type]), 171 # The `shipping_phase` attribute, defaulting to None. This specifies the 172 # release promotion phase that this task belongs to. 173 Required("shipping-phase"): Any( 174 None, 175 "build", 176 "promote", 177 "push", 178 "ship", 179 ), 180 # The `shipping_product` attribute, defaulting to None. This specifies the 181 # release promotion product that this task belongs to. 182 Required("shipping-product"): Any(None, text_type), 183 # The `always-target` attribute will cause the task to be included in the 184 # target_task_graph regardless of filtering. Tasks included in this manner 185 # will be candidates for optimization even when `optimize_target_tasks` is 186 # False, unless the task was also explicitly chosen by the target_tasks 187 # method. 188 Required("always-target"): bool, 189 # Optimization to perform on this task during the optimization phase. 190 # Optimizations are defined in taskcluster/taskgraph/optimize.py. 191 Required("optimization"): OptimizationSchema, 192 # the provisioner-id/worker-type for the task. The following parameters will 193 # be substituted in this string: 194 # {level} -- the scm level of this push 195 "worker-type": text_type, 196 # Whether the job should use sccache compiler caching. 197 Required("use-sccache"): bool, 198 # Set of artifacts relevant to release tasks 199 Optional("release-artifacts"): [text_type], 200 # information specific to the worker implementation that will run this task 201 Optional("worker"): { 202 Required("implementation"): text_type, 203 Extra: object, 204 }, 205 # Override the default priority for the project 206 Optional("priority"): text_type, 207 } 208) 209 210TC_TREEHERDER_SCHEMA_URL = ( 211 "https://github.com/taskcluster/taskcluster-treeherder/" 212 "blob/master/schemas/task-treeherder-config.yml" 213) 214 215 216UNKNOWN_GROUP_NAME = ( 217 "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml" 218) 219 220V2_ROUTE_TEMPLATES = [ 221 "index.{trust-domain}.v2.{project}.latest.{product}.{job-name}", 222 "index.{trust-domain}.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}", 223 "index.{trust-domain}.v2.{project}.pushdate.{build_date}.latest.{product}.{job-name}", 224 "index.{trust-domain}.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}", 225 "index.{trust-domain}.v2.{project}.revision.{branch_rev}.{product}.{job-name}", 226] 227 228# {central, inbound, autoland} write to a "trunk" index prefix. This facilitates 229# walking of tasks with similar configurations. 230V2_TRUNK_ROUTE_TEMPLATES = [ 231 "index.{trust-domain}.v2.trunk.revision.{branch_rev}.{product}.{job-name}", 232] 233 234V2_SHIPPABLE_TEMPLATES = [ 235 "index.{trust-domain}.v2.{project}.shippable.latest.{product}.{job-name}", 236 "index.{trust-domain}.v2.{project}.shippable.{build_date}.revision.{branch_rev}.{product}.{job-name}", # noqa - too long 237 "index.{trust-domain}.v2.{project}.shippable.{build_date}.latest.{product}.{job-name}", 238 "index.{trust-domain}.v2.{project}.shippable.revision.{branch_rev}.{product}.{job-name}", 239] 240 241V2_SHIPPABLE_L10N_TEMPLATES = [ 242 "index.{trust-domain}.v2.{project}.shippable.latest.{product}-l10n.{job-name}.{locale}", 243 "index.{trust-domain}.v2.{project}.shippable.{build_date}.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", # noqa - too long 244 "index.{trust-domain}.v2.{project}.shippable.{build_date}.latest.{product}-l10n.{job-name}.{locale}", # noqa - too long 245 "index.{trust-domain}.v2.{project}.shippable.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", # noqa - too long 246] 247 248V2_L10N_TEMPLATES = [ 249 "index.{trust-domain}.v2.{project}.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", 250 "index.{trust-domain}.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}", # noqa - too long 251 "index.{trust-domain}.v2.{project}.pushlog-id.{pushlog_id}.{product}-l10n.{job-name}.{locale}", 252 "index.{trust-domain}.v2.{project}.latest.{product}-l10n.{job-name}.{locale}", 253] 254 255# This index is specifically for builds that include geckoview releases, 256# so we can hard-code the project to "geckoview" 257V2_GECKOVIEW_RELEASE = "index.{trust-domain}.v2.{project}.geckoview-version.{geckoview-version}.{product}.{job-name}" # noqa - too long 258 259# the roots of the treeherder routes 260TREEHERDER_ROUTE_ROOT = "tc-treeherder" 261 262 263def get_branch_rev(config): 264 return config.params[ 265 "{}head_rev".format(config.graph_config["project-repo-param-prefix"]) 266 ] 267 268 269def get_branch_repo(config): 270 return config.params[ 271 "{}head_repository".format( 272 config.graph_config["project-repo-param-prefix"], 273 ) 274 ] 275 276 277@memoize 278def get_default_priority(graph_config, project): 279 return evaluate_keyed_by( 280 graph_config["task-priority"], "Graph Config", {"project": project} 281 ) 282 283 284# define a collection of payload builders, depending on the worker implementation 285payload_builders = {} 286 287 288@attr.s(frozen=True) 289class PayloadBuilder(object): 290 schema = attr.ib(type=Schema) 291 builder = attr.ib() 292 293 294def payload_builder(name, schema): 295 schema = Schema( 296 {Required("implementation"): name, Optional("os"): text_type} 297 ).extend(schema) 298 299 def wrap(func): 300 payload_builders[name] = PayloadBuilder(schema, func) 301 return func 302 303 return wrap 304 305 306# define a collection of index builders, depending on the type implementation 307index_builders = {} 308 309 310def index_builder(name): 311 def wrap(func): 312 index_builders[name] = func 313 return func 314 315 return wrap 316 317 318UNSUPPORTED_INDEX_PRODUCT_ERROR = """\ 319The gecko-v2 product {product} is not in the list of configured products in 320`taskcluster/ci/config.yml'. 321""" 322 323 324def verify_index(config, index): 325 product = index["product"] 326 if product not in config.graph_config["index"]["products"]: 327 raise Exception(UNSUPPORTED_INDEX_PRODUCT_ERROR.format(product=product)) 328 329 330@payload_builder( 331 "docker-worker", 332 schema={ 333 Required("os"): "linux", 334 # For tasks that will run in docker-worker, this is the 335 # name of the docker image or in-tree docker image to run the task in. If 336 # in-tree, then a dependency will be created automatically. This is 337 # generally `desktop-test`, or an image that acts an awful lot like it. 338 Required("docker-image"): Any( 339 # a raw Docker image path (repo/image:tag) 340 text_type, 341 # an in-tree generated docker image (from `taskcluster/docker/<name>`) 342 {"in-tree": text_type}, 343 # an indexed docker image 344 {"indexed": text_type}, 345 ), 346 # worker features that should be enabled 347 Required("chain-of-trust"): bool, 348 Required("taskcluster-proxy"): bool, 349 Required("allow-ptrace"): bool, 350 Required("loopback-video"): bool, 351 Required("loopback-audio"): bool, 352 Required("docker-in-docker"): bool, # (aka 'dind') 353 Required("privileged"): bool, 354 # Paths to Docker volumes. 355 # 356 # For in-tree Docker images, volumes can be parsed from Dockerfile. 357 # This only works for the Dockerfile itself: if a volume is defined in 358 # a base image, it will need to be declared here. Out-of-tree Docker 359 # images will also require explicit volume annotation. 360 # 361 # Caches are often mounted to the same path as Docker volumes. In this 362 # case, they take precedence over a Docker volume. But a volume still 363 # needs to be declared for the path. 364 Optional("volumes"): [text_type], 365 Optional( 366 "required-volumes", 367 description=( 368 "Paths that are required to be volumes for performance reasons. " 369 "For in-tree images, these paths will be checked to verify that they " 370 "are defined as volumes." 371 ), 372 ): [text_type], 373 # caches to set up for the task 374 Optional("caches"): [ 375 { 376 # only one type is supported by any of the workers right now 377 "type": "persistent", 378 # name of the cache, allowing re-use by subsequent tasks naming the 379 # same cache 380 "name": text_type, 381 # location in the task image where the cache will be mounted 382 "mount-point": text_type, 383 # Whether the cache is not used in untrusted environments 384 # (like the Try repo). 385 Optional("skip-untrusted"): bool, 386 } 387 ], 388 # artifacts to extract from the task image after completion 389 Optional("artifacts"): [ 390 { 391 # type of artifact -- simple file, or recursive directory 392 "type": Any("file", "directory"), 393 # task image path from which to read artifact 394 "path": text_type, 395 # name of the produced artifact (root of the names for 396 # type=directory) 397 "name": text_type, 398 } 399 ], 400 # environment variables 401 Required("env"): {text_type: taskref_or_string}, 402 # the command to run; if not given, docker-worker will default to the 403 # command in the docker image 404 Optional("command"): [taskref_or_string], 405 # the maximum time to run, in seconds 406 Required("max-run-time"): int, 407 # the exit status code(s) that indicates the task should be retried 408 Optional("retry-exit-status"): [int], 409 # the exit status code(s) that indicates the caches used by the task 410 # should be purged 411 Optional("purge-caches-exit-status"): [int], 412 # Wether any artifacts are assigned to this worker 413 Optional("skip-artifacts"): bool, 414 }, 415) 416def build_docker_worker_payload(config, task, task_def): 417 worker = task["worker"] 418 level = int(config.params["level"]) 419 420 image = worker["docker-image"] 421 if isinstance(image, dict): 422 if "in-tree" in image: 423 name = image["in-tree"] 424 docker_image_task = "docker-image-" + image["in-tree"] 425 task.setdefault("dependencies", {})["docker-image"] = docker_image_task 426 427 image = { 428 "path": "public/image.tar.zst", 429 "taskId": {"task-reference": "<docker-image>"}, 430 "type": "task-image", 431 } 432 433 # Find VOLUME in Dockerfile. 434 volumes = dockerutil.parse_volumes(name) 435 for v in sorted(volumes): 436 if v in worker["volumes"]: 437 raise Exception( 438 "volume %s already defined; " 439 "if it is defined in a Dockerfile, " 440 "it does not need to be specified in the " 441 "worker definition" % v 442 ) 443 444 worker["volumes"].append(v) 445 446 elif "indexed" in image: 447 image = { 448 "path": "public/image.tar.zst", 449 "namespace": image["indexed"], 450 "type": "indexed-image", 451 } 452 else: 453 raise Exception("unknown docker image type") 454 455 features = {} 456 457 if worker.get("taskcluster-proxy"): 458 features["taskclusterProxy"] = True 459 460 if worker.get("allow-ptrace"): 461 features["allowPtrace"] = True 462 task_def["scopes"].append("docker-worker:feature:allowPtrace") 463 464 if worker.get("chain-of-trust"): 465 features["chainOfTrust"] = True 466 467 if worker.get("docker-in-docker"): 468 features["dind"] = True 469 470 if task.get("use-sccache"): 471 features["taskclusterProxy"] = True 472 task_def["scopes"].append( 473 "assume:project:taskcluster:{trust_domain}:level-{level}-sccache-buckets".format( 474 trust_domain=config.graph_config["trust-domain"], 475 level=config.params["level"], 476 ) 477 ) 478 worker["env"]["USE_SCCACHE"] = "1" 479 worker["env"]["SCCACHE_GCS_PROJECT"] = SCCACHE_GCS_PROJECT 480 # Disable sccache idle shutdown. 481 worker["env"]["SCCACHE_IDLE_TIMEOUT"] = "0" 482 else: 483 worker["env"]["SCCACHE_DISABLE"] = "1" 484 485 capabilities = {} 486 487 for lo in "audio", "video": 488 if worker.get("loopback-" + lo): 489 capitalized = "loopback" + lo.capitalize() 490 devices = capabilities.setdefault("devices", {}) 491 devices[capitalized] = True 492 task_def["scopes"].append("docker-worker:capability:device:" + capitalized) 493 494 if worker.get("privileged"): 495 capabilities["privileged"] = True 496 task_def["scopes"].append("docker-worker:capability:privileged") 497 498 task_def["payload"] = payload = { 499 "image": image, 500 "env": worker["env"], 501 } 502 if "command" in worker: 503 payload["command"] = worker["command"] 504 505 if "max-run-time" in worker: 506 payload["maxRunTime"] = worker["max-run-time"] 507 508 run_task = payload.get("command", [""])[0].endswith("run-task") 509 510 # run-task exits EXIT_PURGE_CACHES if there is a problem with caches. 511 # Automatically retry the tasks and purge caches if we see this exit 512 # code. 513 # TODO move this closer to code adding run-task once bug 1469697 is 514 # addressed. 515 if run_task: 516 worker.setdefault("retry-exit-status", []).append(72) 517 worker.setdefault("purge-caches-exit-status", []).append(72) 518 519 payload["onExitStatus"] = {} 520 if "retry-exit-status" in worker: 521 payload["onExitStatus"]["retry"] = worker["retry-exit-status"] 522 if "purge-caches-exit-status" in worker: 523 payload["onExitStatus"]["purgeCaches"] = worker["purge-caches-exit-status"] 524 525 if "artifacts" in worker: 526 artifacts = {} 527 for artifact in worker["artifacts"]: 528 artifacts[artifact["name"]] = { 529 "path": artifact["path"], 530 "type": artifact["type"], 531 "expires": task_def["expires"], # always expire with the task 532 } 533 payload["artifacts"] = artifacts 534 535 if isinstance(worker.get("docker-image"), text_type): 536 out_of_tree_image = worker["docker-image"] 537 else: 538 out_of_tree_image = None 539 image = worker.get("docker-image", {}).get("in-tree") 540 541 if "caches" in worker: 542 caches = {} 543 544 # run-task knows how to validate caches. 545 # 546 # To help ensure new run-task features and bug fixes don't interfere 547 # with existing caches, we seed the hash of run-task into cache names. 548 # So, any time run-task changes, we should get a fresh set of caches. 549 # This means run-task can make changes to cache interaction at any time 550 # without regards for backwards or future compatibility. 551 # 552 # But this mechanism only works for in-tree Docker images that are built 553 # with the current run-task! For out-of-tree Docker images, we have no 554 # way of knowing their content of run-task. So, in addition to varying 555 # cache names by the contents of run-task, we also take the Docker image 556 # name into consideration. This means that different Docker images will 557 # never share the same cache. This is a bit unfortunate. But it is the 558 # safest thing to do. Fortunately, most images are defined in-tree. 559 # 560 # For out-of-tree Docker images, we don't strictly need to incorporate 561 # the run-task content into the cache name. However, doing so preserves 562 # the mechanism whereby changing run-task results in new caches 563 # everywhere. 564 565 # As an additional mechanism to force the use of different caches, the 566 # string literal in the variable below can be changed. This is 567 # preferred to changing run-task because it doesn't require images 568 # to be rebuilt. 569 cache_version = "v3" 570 571 if run_task: 572 suffix = "{}-{}".format(cache_version, _run_task_suffix()) 573 574 if out_of_tree_image: 575 name_hash = hashlib.sha256( 576 six.ensure_binary(out_of_tree_image) 577 ).hexdigest() 578 suffix += name_hash[0:12] 579 580 else: 581 suffix = cache_version 582 583 skip_untrusted = config.params.is_try() or level == 1 584 585 for cache in worker["caches"]: 586 # Some caches aren't enabled in environments where we can't 587 # guarantee certain behavior. Filter those out. 588 if cache.get("skip-untrusted") and skip_untrusted: 589 continue 590 591 name = "{trust_domain}-level-{level}-{name}-{suffix}".format( 592 trust_domain=config.graph_config["trust-domain"], 593 level=config.params["level"], 594 name=cache["name"], 595 suffix=suffix, 596 ) 597 598 caches[name] = cache["mount-point"] 599 task_def["scopes"].append("docker-worker:cache:%s" % name) 600 601 # Assertion: only run-task is interested in this. 602 if run_task: 603 payload["env"]["TASKCLUSTER_CACHES"] = ";".join(sorted(caches.values())) 604 605 payload["cache"] = caches 606 607 # And send down volumes information to run-task as well. 608 if run_task and worker.get("volumes"): 609 payload["env"]["TASKCLUSTER_VOLUMES"] = ";".join( 610 [six.ensure_text(s) for s in sorted(worker["volumes"])] 611 ) 612 613 if payload.get("cache") and skip_untrusted: 614 payload["env"]["TASKCLUSTER_UNTRUSTED_CACHES"] = "1" 615 616 if features: 617 payload["features"] = features 618 if capabilities: 619 payload["capabilities"] = capabilities 620 621 check_caches_are_volumes(task) 622 check_required_volumes(task) 623 624 625@payload_builder( 626 "generic-worker", 627 schema={ 628 Required("os"): Any("windows", "macosx", "linux", "linux-bitbar"), 629 # see http://schemas.taskcluster.net/generic-worker/v1/payload.json 630 # and https://docs.taskcluster.net/reference/workers/generic-worker/payload 631 # command is a list of commands to run, sequentially 632 # on Windows, each command is a string, on OS X and Linux, each command is 633 # a string array 634 Required("command"): Any( 635 [taskref_or_string], [[taskref_or_string]] # Windows # Linux / OS X 636 ), 637 # artifacts to extract from the task image after completion; note that artifacts 638 # for the generic worker cannot have names 639 Optional("artifacts"): [ 640 { 641 # type of artifact -- simple file, or recursive directory 642 "type": Any("file", "directory"), 643 # filesystem path from which to read artifact 644 "path": text_type, 645 # if not specified, path is used for artifact name 646 Optional("name"): text_type, 647 } 648 ], 649 # Directories and/or files to be mounted. 650 # The actual allowed combinations are stricter than the model below, 651 # but this provides a simple starting point. 652 # See https://docs.taskcluster.net/reference/workers/generic-worker/payload 653 Optional("mounts"): [ 654 { 655 # A unique name for the cache volume, implies writable cache directory 656 # (otherwise mount is a read-only file or directory). 657 Optional("cache-name"): text_type, 658 # Optional content for pre-loading cache, or mandatory content for 659 # read-only file or directory. Pre-loaded content can come from either 660 # a task artifact or from a URL. 661 Optional("content"): { 662 # *** Either (artifact and task-id) or url must be specified. *** 663 # Artifact name that contains the content. 664 Optional("artifact"): text_type, 665 # Task ID that has the artifact that contains the content. 666 Optional("task-id"): taskref_or_string, 667 # URL that supplies the content in response to an unauthenticated 668 # GET request. 669 Optional("url"): text_type, 670 }, 671 # *** Either file or directory must be specified. *** 672 # If mounting a cache or read-only directory, the filesystem location of 673 # the directory should be specified as a relative path to the task 674 # directory here. 675 Optional("directory"): text_type, 676 # If mounting a file, specify the relative path within the task 677 # directory to mount the file (the file will be read only). 678 Optional("file"): text_type, 679 # Required if and only if `content` is specified and mounting a 680 # directory (not a file). This should be the archive format of the 681 # content (either pre-loaded cache or read-only directory). 682 Optional("format"): Any("rar", "tar.bz2", "tar.gz", "zip"), 683 } 684 ], 685 # environment variables 686 Required("env"): {text_type: taskref_or_string}, 687 # the maximum time to run, in seconds 688 Required("max-run-time"): int, 689 # os user groups for test task workers 690 Optional("os-groups"): [text_type], 691 # feature for test task to run as administarotr 692 Optional("run-as-administrator"): bool, 693 # optional features 694 Required("chain-of-trust"): bool, 695 Optional("taskcluster-proxy"): bool, 696 # the exit status code(s) that indicates the task should be retried 697 Optional("retry-exit-status"): [int], 698 # Wether any artifacts are assigned to this worker 699 Optional("skip-artifacts"): bool, 700 }, 701) 702def build_generic_worker_payload(config, task, task_def): 703 worker = task["worker"] 704 features = {} 705 706 task_def["payload"] = { 707 "command": worker["command"], 708 "maxRunTime": worker["max-run-time"], 709 } 710 711 if worker["os"] == "windows": 712 task_def["payload"]["onExitStatus"] = { 713 "retry": [ 714 # These codes (on windows) indicate a process interruption, 715 # rather than a task run failure. See bug 1544403. 716 1073807364, # process force-killed due to system shutdown 717 3221225786, # sigint (any interrupt) 718 ] 719 } 720 if "retry-exit-status" in worker: 721 task_def["payload"].setdefault("onExitStatus", {}).setdefault( 722 "retry", [] 723 ).extend(worker["retry-exit-status"]) 724 if worker["os"] == "linux-bitbar": 725 task_def["payload"].setdefault("onExitStatus", {}).setdefault("retry", []) 726 # exit code 4 is used to indicate an intermittent android device error 727 if 4 not in task_def["payload"]["onExitStatus"]["retry"]: 728 task_def["payload"]["onExitStatus"]["retry"].extend([4]) 729 730 env = worker.get("env", {}) 731 732 if task.get("use-sccache"): 733 features["taskclusterProxy"] = True 734 task_def["scopes"].append( 735 "assume:project:taskcluster:{trust_domain}:level-{level}-sccache-buckets".format( 736 trust_domain=config.graph_config["trust-domain"], 737 level=config.params["level"], 738 ) 739 ) 740 env["USE_SCCACHE"] = "1" 741 worker["env"]["SCCACHE_GCS_PROJECT"] = SCCACHE_GCS_PROJECT 742 # Disable sccache idle shutdown. 743 env["SCCACHE_IDLE_TIMEOUT"] = "0" 744 else: 745 env["SCCACHE_DISABLE"] = "1" 746 747 if env: 748 task_def["payload"]["env"] = env 749 750 artifacts = [] 751 752 for artifact in worker.get("artifacts", []): 753 a = { 754 "path": artifact["path"], 755 "type": artifact["type"], 756 } 757 if "name" in artifact: 758 a["name"] = artifact["name"] 759 artifacts.append(a) 760 761 if artifacts: 762 task_def["payload"]["artifacts"] = artifacts 763 764 # Need to copy over mounts, but rename keys to respect naming convention 765 # * 'cache-name' -> 'cacheName' 766 # * 'task-id' -> 'taskId' 767 # All other key names are already suitable, and don't need renaming. 768 mounts = deepcopy(worker.get("mounts", [])) 769 for mount in mounts: 770 if "cache-name" in mount: 771 mount["cacheName"] = "{trust_domain}-level-{level}-{name}".format( 772 trust_domain=config.graph_config["trust-domain"], 773 level=config.params["level"], 774 name=mount.pop("cache-name"), 775 ) 776 task_def["scopes"].append( 777 "generic-worker:cache:{}".format(mount["cacheName"]) 778 ) 779 if "content" in mount: 780 if "task-id" in mount["content"]: 781 mount["content"]["taskId"] = mount["content"].pop("task-id") 782 if "artifact" in mount["content"]: 783 if not mount["content"]["artifact"].startswith("public/"): 784 task_def["scopes"].append( 785 "queue:get-artifact:{}".format(mount["content"]["artifact"]) 786 ) 787 788 if mounts: 789 task_def["payload"]["mounts"] = mounts 790 791 if worker.get("os-groups"): 792 task_def["payload"]["osGroups"] = worker["os-groups"] 793 task_def["scopes"].extend( 794 [ 795 "generic-worker:os-group:{}/{}".format(task["worker-type"], group) 796 for group in worker["os-groups"] 797 ] 798 ) 799 800 if worker.get("chain-of-trust"): 801 features["chainOfTrust"] = True 802 803 if worker.get("taskcluster-proxy"): 804 features["taskclusterProxy"] = True 805 806 if worker.get("run-as-administrator", False): 807 features["runAsAdministrator"] = True 808 task_def["scopes"].append( 809 "generic-worker:run-as-administrator:{}".format(task["worker-type"]), 810 ) 811 812 if features: 813 task_def["payload"]["features"] = features 814 815 816@payload_builder( 817 "scriptworker-signing", 818 schema={ 819 # the maximum time to run, in seconds 820 Required("max-run-time"): int, 821 # list of artifact URLs for the artifacts that should be signed 822 Required("upstream-artifacts"): [ 823 { 824 # taskId of the task with the artifact 825 Required("taskId"): taskref_or_string, 826 # type of signing task (for CoT) 827 Required("taskType"): text_type, 828 # Paths to the artifacts to sign 829 Required("paths"): [text_type], 830 # Signing formats to use on each of the paths 831 Required("formats"): [text_type], 832 Optional("singleFileGlobs"): [text_type], 833 } 834 ], 835 # behavior for mac iscript 836 Optional("mac-behavior"): Any( 837 "mac_notarize_part_1", 838 "mac_notarize_part_3", 839 "mac_sign_and_pkg", 840 "mac_geckodriver", 841 "mac_single_file", 842 ), 843 Optional("entitlements-url"): text_type, 844 }, 845) 846def build_scriptworker_signing_payload(config, task, task_def): 847 worker = task["worker"] 848 849 task_def["payload"] = { 850 "maxRunTime": worker["max-run-time"], 851 "upstreamArtifacts": worker["upstream-artifacts"], 852 } 853 if worker.get("mac-behavior"): 854 task_def["payload"]["behavior"] = worker["mac-behavior"] 855 if worker.get("entitlements-url"): 856 task_def["payload"]["entitlements-url"] = worker["entitlements-url"] 857 artifacts = set(task.get("release-artifacts", [])) 858 for upstream_artifact in worker["upstream-artifacts"]: 859 for path in upstream_artifact["paths"]: 860 artifacts.update( 861 get_signed_artifacts( 862 input=path, 863 formats=upstream_artifact["formats"], 864 behavior=worker.get("mac-behavior"), 865 ) 866 ) 867 task["release-artifacts"] = list(artifacts) 868 869 870@payload_builder( 871 "notarization-poller", 872 schema={ 873 Required("uuid-manifest"): taskref_or_string, 874 }, 875) 876def notarization_poller_payload(config, task, task_def): 877 worker = task["worker"] 878 task_def["payload"] = {"uuid_manifest": worker["uuid-manifest"]} 879 880 881@payload_builder( 882 "beetmover", 883 schema={ 884 # the maximum time to run, in seconds 885 Required("max-run-time"): int, 886 # locale key, if this is a locale beetmover job 887 Optional("locale"): text_type, 888 Optional("partner-public"): bool, 889 Required("release-properties"): { 890 "app-name": text_type, 891 "app-version": text_type, 892 "branch": text_type, 893 "build-id": text_type, 894 "hash-type": text_type, 895 "platform": text_type, 896 }, 897 # list of artifact URLs for the artifacts that should be beetmoved 898 Required("upstream-artifacts"): [ 899 { 900 # taskId of the task with the artifact 901 Required("taskId"): taskref_or_string, 902 # type of signing task (for CoT) 903 Required("taskType"): text_type, 904 # Paths to the artifacts to sign 905 Required("paths"): [text_type], 906 # locale is used to map upload path and allow for duplicate simple names 907 Required("locale"): text_type, 908 } 909 ], 910 Optional("artifact-map"): object, 911 }, 912) 913def build_beetmover_payload(config, task, task_def): 914 worker = task["worker"] 915 release_config = get_release_config(config) 916 release_properties = worker["release-properties"] 917 918 task_def["payload"] = { 919 "maxRunTime": worker["max-run-time"], 920 "releaseProperties": { 921 "appName": release_properties["app-name"], 922 "appVersion": release_properties["app-version"], 923 "branch": release_properties["branch"], 924 "buildid": release_properties["build-id"], 925 "hashType": release_properties["hash-type"], 926 "platform": release_properties["platform"], 927 }, 928 "upload_date": config.params["build_date"], 929 "upstreamArtifacts": worker["upstream-artifacts"], 930 } 931 if worker.get("locale"): 932 task_def["payload"]["locale"] = worker["locale"] 933 if worker.get("artifact-map"): 934 task_def["payload"]["artifactMap"] = worker["artifact-map"] 935 if worker.get("partner-public"): 936 task_def["payload"]["is_partner_repack_public"] = worker["partner-public"] 937 if release_config: 938 task_def["payload"].update(release_config) 939 940 941@payload_builder( 942 "beetmover-push-to-release", 943 schema={ 944 # the maximum time to run, in seconds 945 Required("max-run-time"): int, 946 Required("product"): text_type, 947 }, 948) 949def build_beetmover_push_to_release_payload(config, task, task_def): 950 worker = task["worker"] 951 release_config = get_release_config(config) 952 partners = [ 953 "{}/{}".format(p, s) for p, s, _ in get_partners_to_be_published(config) 954 ] 955 956 task_def["payload"] = { 957 "maxRunTime": worker["max-run-time"], 958 "product": worker["product"], 959 "version": release_config["version"], 960 "build_number": release_config["build_number"], 961 "partners": partners, 962 } 963 964 965@payload_builder( 966 "beetmover-maven", 967 schema={ 968 Required("max-run-time"): int, 969 Required("release-properties"): { 970 "app-name": text_type, 971 "app-version": text_type, 972 "branch": text_type, 973 "build-id": text_type, 974 "artifact-id": text_type, 975 "hash-type": text_type, 976 "platform": text_type, 977 }, 978 Required("upstream-artifacts"): [ 979 { 980 Required("taskId"): taskref_or_string, 981 Required("taskType"): text_type, 982 Required("paths"): [text_type], 983 Optional("zipExtract"): bool, 984 } 985 ], 986 Optional("artifact-map"): object, 987 }, 988) 989def build_beetmover_maven_payload(config, task, task_def): 990 build_beetmover_payload(config, task, task_def) 991 992 task_def["payload"]["artifact_id"] = task["worker"]["release-properties"][ 993 "artifact-id" 994 ] 995 if task["worker"].get("artifact-map"): 996 task_def["payload"]["artifactMap"] = task["worker"]["artifact-map"] 997 998 task_def["payload"]["version"] = _compute_geckoview_version( 999 task["worker"]["release-properties"]["app-version"], 1000 task["worker"]["release-properties"]["build-id"], 1001 ) 1002 1003 del task_def["payload"]["releaseProperties"]["hashType"] 1004 del task_def["payload"]["releaseProperties"]["platform"] 1005 1006 1007@payload_builder( 1008 "balrog", 1009 schema={ 1010 Required("balrog-action"): Any(*BALROG_ACTIONS), 1011 Optional("product"): text_type, 1012 Optional("platforms"): [text_type], 1013 Optional("release-eta"): text_type, 1014 Optional("channel-names"): optionally_keyed_by("release-type", [text_type]), 1015 Optional("require-mirrors"): bool, 1016 Optional("publish-rules"): optionally_keyed_by( 1017 "release-type", "release-level", [int] 1018 ), 1019 Optional("rules-to-update"): optionally_keyed_by( 1020 "release-type", "release-level", [text_type] 1021 ), 1022 Optional("archive-domain"): optionally_keyed_by("release-level", text_type), 1023 Optional("download-domain"): optionally_keyed_by("release-level", text_type), 1024 Optional("blob-suffix"): text_type, 1025 Optional("complete-mar-filename-pattern"): text_type, 1026 Optional("complete-mar-bouncer-product-pattern"): text_type, 1027 Optional("update-line"): object, 1028 Optional("suffixes"): [text_type], 1029 Optional("background-rate"): optionally_keyed_by( 1030 "release-type", "beta-number", Any(int, None) 1031 ), 1032 Optional("force-fallback-mapping-update"): optionally_keyed_by( 1033 "release-type", "beta-number", bool 1034 ), 1035 # list of artifact URLs for the artifacts that should be beetmoved 1036 Optional("upstream-artifacts"): [ 1037 { 1038 # taskId of the task with the artifact 1039 Required("taskId"): taskref_or_string, 1040 # type of signing task (for CoT) 1041 Required("taskType"): text_type, 1042 # Paths to the artifacts to sign 1043 Required("paths"): [text_type], 1044 } 1045 ], 1046 }, 1047) 1048def build_balrog_payload(config, task, task_def): 1049 worker = task["worker"] 1050 release_config = get_release_config(config) 1051 beta_number = None 1052 if "b" in release_config["version"]: 1053 beta_number = release_config["version"].split("b")[-1] 1054 1055 task_def["payload"] = { 1056 "behavior": worker["balrog-action"], 1057 } 1058 1059 if ( 1060 worker["balrog-action"] == "submit-locale" 1061 or worker["balrog-action"] == "v2-submit-locale" 1062 ): 1063 task_def["payload"].update( 1064 { 1065 "upstreamArtifacts": worker["upstream-artifacts"], 1066 "suffixes": worker["suffixes"], 1067 } 1068 ) 1069 else: 1070 for prop in ( 1071 "archive-domain", 1072 "channel-names", 1073 "download-domain", 1074 "publish-rules", 1075 "rules-to-update", 1076 "background-rate", 1077 "force-fallback-mapping-update", 1078 ): 1079 if prop in worker: 1080 resolve_keyed_by( 1081 worker, 1082 prop, 1083 task["description"], 1084 **{ 1085 "release-type": config.params["release_type"], 1086 "release-level": config.params.release_level(), 1087 "beta-number": beta_number, 1088 } 1089 ) 1090 task_def["payload"].update( 1091 { 1092 "build_number": release_config["build_number"], 1093 "product": worker["product"], 1094 "version": release_config["version"], 1095 } 1096 ) 1097 for prop in ( 1098 "blob-suffix", 1099 "complete-mar-filename-pattern", 1100 "complete-mar-bouncer-product-pattern", 1101 ): 1102 if prop in worker: 1103 task_def["payload"][prop.replace("-", "_")] = worker[prop] 1104 if ( 1105 worker["balrog-action"] == "submit-toplevel" 1106 or worker["balrog-action"] == "v2-submit-toplevel" 1107 ): 1108 task_def["payload"].update( 1109 { 1110 "app_version": release_config["appVersion"], 1111 "archive_domain": worker["archive-domain"], 1112 "channel_names": worker["channel-names"], 1113 "download_domain": worker["download-domain"], 1114 "partial_versions": release_config.get("partial_versions", ""), 1115 "platforms": worker["platforms"], 1116 "rules_to_update": worker["rules-to-update"], 1117 "require_mirrors": worker["require-mirrors"], 1118 "update_line": worker["update-line"], 1119 } 1120 ) 1121 else: # schedule / ship 1122 task_def["payload"].update( 1123 { 1124 "publish_rules": worker["publish-rules"], 1125 "release_eta": worker.get( 1126 "release-eta", config.params.get("release_eta") 1127 ) 1128 or "", 1129 } 1130 ) 1131 if worker.get("force-fallback-mapping-update"): 1132 task_def["payload"]["force_fallback_mapping_update"] = worker[ 1133 "force-fallback-mapping-update" 1134 ] 1135 if worker.get("background-rate"): 1136 task_def["payload"]["background_rate"] = worker["background-rate"] 1137 1138 1139@payload_builder( 1140 "bouncer-aliases", 1141 schema={ 1142 Required("entries"): object, 1143 }, 1144) 1145def build_bouncer_aliases_payload(config, task, task_def): 1146 worker = task["worker"] 1147 1148 task_def["payload"] = {"aliases_entries": worker["entries"]} 1149 1150 1151@payload_builder( 1152 "bouncer-locations", 1153 schema={ 1154 Required("implementation"): "bouncer-locations", 1155 Required("bouncer-products"): [text_type], 1156 }, 1157) 1158def build_bouncer_locations_payload(config, task, task_def): 1159 worker = task["worker"] 1160 release_config = get_release_config(config) 1161 1162 task_def["payload"] = { 1163 "bouncer_products": worker["bouncer-products"], 1164 "version": release_config["version"], 1165 "product": task["shipping-product"], 1166 } 1167 1168 1169@payload_builder( 1170 "bouncer-submission", 1171 schema={ 1172 Required("locales"): [text_type], 1173 Required("entries"): object, 1174 }, 1175) 1176def build_bouncer_submission_payload(config, task, task_def): 1177 worker = task["worker"] 1178 1179 task_def["payload"] = { 1180 "locales": worker["locales"], 1181 "submission_entries": worker["entries"], 1182 } 1183 1184 1185@payload_builder( 1186 "push-flatpak", 1187 schema={ 1188 Required("channel"): text_type, 1189 Required("upstream-artifacts"): [ 1190 { 1191 Required("taskId"): taskref_or_string, 1192 Required("taskType"): text_type, 1193 Required("paths"): [text_type], 1194 } 1195 ], 1196 }, 1197) 1198def build_push_flatpak_payload(config, task, task_def): 1199 worker = task["worker"] 1200 1201 task_def["payload"] = { 1202 "channel": worker["channel"], 1203 "upstreamArtifacts": worker["upstream-artifacts"], 1204 } 1205 1206 1207@payload_builder( 1208 "shipit-shipped", 1209 schema={ 1210 Required("release-name"): text_type, 1211 }, 1212) 1213def build_ship_it_shipped_payload(config, task, task_def): 1214 worker = task["worker"] 1215 1216 task_def["payload"] = {"release_name": worker["release-name"]} 1217 1218 1219@payload_builder( 1220 "shipit-maybe-release", 1221 schema={ 1222 Required("phase"): text_type, 1223 }, 1224) 1225def build_ship_it_maybe_release_payload(config, task, task_def): 1226 # expect branch name, including path 1227 branch = config.params["head_repository"][len("https://hg.mozilla.org/") :] 1228 # 'version' is e.g. '71.0b13' (app_version doesn't have beta number) 1229 version = config.params["version"] 1230 1231 task_def["payload"] = { 1232 "product": task["shipping-product"], 1233 "branch": branch, 1234 "phase": task["worker"]["phase"], 1235 "version": version, 1236 "cron_revision": config.params["head_rev"], 1237 } 1238 1239 1240@payload_builder( 1241 "push-addons", 1242 schema={ 1243 Required("channel"): Any("listed", "unlisted"), 1244 Required("upstream-artifacts"): [ 1245 { 1246 Required("taskId"): taskref_or_string, 1247 Required("taskType"): text_type, 1248 Required("paths"): [text_type], 1249 } 1250 ], 1251 }, 1252) 1253def build_push_addons_payload(config, task, task_def): 1254 worker = task["worker"] 1255 1256 task_def["payload"] = { 1257 "channel": worker["channel"], 1258 "upstreamArtifacts": worker["upstream-artifacts"], 1259 } 1260 1261 1262@payload_builder( 1263 "treescript", 1264 schema={ 1265 Required("tags"): [Any("buildN", "release", None)], 1266 Required("bump"): bool, 1267 Optional("bump-files"): [text_type], 1268 Optional("repo-param-prefix"): text_type, 1269 Optional("dontbuild"): bool, 1270 Optional("ignore-closed-tree"): bool, 1271 Optional("force-dry-run"): bool, 1272 Optional("push"): bool, 1273 Optional("source-repo"): text_type, 1274 Optional("ssh-user"): text_type, 1275 Optional("l10n-bump-info"): { 1276 Required("name"): text_type, 1277 Required("path"): text_type, 1278 Required("version-path"): text_type, 1279 Optional("l10n-repo-url"): text_type, 1280 Optional("ignore-config"): object, 1281 Required("platform-configs"): [ 1282 { 1283 Required("platforms"): [text_type], 1284 Required("path"): text_type, 1285 Optional("format"): text_type, 1286 } 1287 ], 1288 }, 1289 Optional("merge-info"): object, 1290 }, 1291) 1292def build_treescript_payload(config, task, task_def): 1293 worker = task["worker"] 1294 release_config = get_release_config(config) 1295 1296 task_def["payload"] = {"actions": []} 1297 actions = task_def["payload"]["actions"] 1298 if worker["tags"]: 1299 tag_names = [] 1300 product = task["shipping-product"].upper() 1301 version = release_config["version"].replace(".", "_") 1302 buildnum = release_config["build_number"] 1303 if "buildN" in worker["tags"]: 1304 tag_names.extend( 1305 [ 1306 "{}_{}_BUILD{}".format(product, version, buildnum), 1307 ] 1308 ) 1309 if "release" in worker["tags"]: 1310 tag_names.extend(["{}_{}_RELEASE".format(product, version)]) 1311 tag_info = { 1312 "tags": tag_names, 1313 "revision": config.params[ 1314 "{}head_rev".format(worker.get("repo-param-prefix", "")) 1315 ], 1316 } 1317 task_def["payload"]["tag_info"] = tag_info 1318 actions.append("tag") 1319 1320 if worker["bump"]: 1321 if not worker["bump-files"]: 1322 raise Exception("Version Bump requested without bump-files") 1323 1324 bump_info = {} 1325 bump_info["next_version"] = release_config["next_version"] 1326 bump_info["files"] = worker["bump-files"] 1327 task_def["payload"]["version_bump_info"] = bump_info 1328 actions.append("version_bump") 1329 1330 if worker.get("l10n-bump-info"): 1331 l10n_bump_info = {} 1332 for k, v in worker["l10n-bump-info"].items(): 1333 l10n_bump_info[k.replace("-", "_")] = worker["l10n-bump-info"][k] 1334 task_def["payload"]["l10n_bump_info"] = [l10n_bump_info] 1335 actions.append("l10n_bump") 1336 1337 if worker.get("merge-info"): 1338 merge_info = { 1339 merge_param_name.replace("-", "_"): merge_param_value 1340 for merge_param_name, merge_param_value in worker["merge-info"].items() 1341 if merge_param_name != "version-files" 1342 } 1343 merge_info["version_files"] = [ 1344 { 1345 file_param_name.replace("-", "_"): file_param_value 1346 for file_param_name, file_param_value in file_entry.items() 1347 } 1348 for file_entry in worker["merge-info"]["version-files"] 1349 ] 1350 task_def["payload"]["merge_info"] = merge_info 1351 actions.append("merge_day") 1352 1353 if worker["push"]: 1354 actions.append("push") 1355 1356 if worker.get("force-dry-run"): 1357 task_def["payload"]["dry_run"] = True 1358 1359 if worker.get("dontbuild"): 1360 task_def["payload"]["dontbuild"] = True 1361 1362 if worker.get("ignore-closed-tree") is not None: 1363 task_def["payload"]["ignore_closed_tree"] = worker["ignore-closed-tree"] 1364 1365 if worker.get("source-repo"): 1366 task_def["payload"]["source_repo"] = worker["source-repo"] 1367 1368 if worker.get("ssh-user"): 1369 task_def["payload"]["ssh_user"] = worker["ssh-user"] 1370 1371 1372@payload_builder( 1373 "invalid", 1374 schema={ 1375 # an invalid task is one which should never actually be created; this is used in 1376 # release automation on branches where the task just doesn't make sense 1377 Extra: object, 1378 }, 1379) 1380def build_invalid_payload(config, task, task_def): 1381 task_def["payload"] = "invalid task - should never be created" 1382 1383 1384@payload_builder( 1385 "always-optimized", 1386 schema={ 1387 Extra: object, 1388 }, 1389) 1390@payload_builder("succeed", schema={}) 1391def build_dummy_payload(config, task, task_def): 1392 task_def["payload"] = {} 1393 1394 1395transforms = TransformSequence() 1396 1397 1398@transforms.add 1399def set_implementation(config, tasks): 1400 """ 1401 Set the worker implementation based on the worker-type alias. 1402 """ 1403 for task in tasks: 1404 if "implementation" in task["worker"]: 1405 yield task 1406 continue 1407 1408 impl, os = worker_type_implementation(config.graph_config, task["worker-type"]) 1409 1410 tags = task.setdefault("tags", {}) 1411 tags["worker-implementation"] = impl 1412 if os: 1413 task["tags"]["os"] = os 1414 worker = task.setdefault("worker", {}) 1415 worker["implementation"] = impl 1416 if os: 1417 worker["os"] = os 1418 1419 yield task 1420 1421 1422@transforms.add 1423def set_defaults(config, tasks): 1424 for task in tasks: 1425 task.setdefault("shipping-phase", None) 1426 task.setdefault("shipping-product", None) 1427 task.setdefault("always-target", False) 1428 task.setdefault("optimization", None) 1429 task.setdefault("use-sccache", False) 1430 1431 worker = task["worker"] 1432 if worker["implementation"] in ("docker-worker",): 1433 worker.setdefault("chain-of-trust", False) 1434 worker.setdefault("taskcluster-proxy", False) 1435 worker.setdefault("allow-ptrace", True) 1436 worker.setdefault("loopback-video", False) 1437 worker.setdefault("loopback-audio", False) 1438 worker.setdefault("docker-in-docker", False) 1439 worker.setdefault("privileged", False) 1440 worker.setdefault("volumes", []) 1441 worker.setdefault("env", {}) 1442 if "caches" in worker: 1443 for c in worker["caches"]: 1444 c.setdefault("skip-untrusted", False) 1445 elif worker["implementation"] == "generic-worker": 1446 worker.setdefault("env", {}) 1447 worker.setdefault("os-groups", []) 1448 if worker["os-groups"] and worker["os"] != "windows": 1449 raise Exception( 1450 "os-groups feature of generic-worker is only supported on " 1451 "Windows, not on {}".format(worker["os"]) 1452 ) 1453 worker.setdefault("chain-of-trust", False) 1454 elif worker["implementation"] in ( 1455 "scriptworker-signing", 1456 "beetmover", 1457 "beetmover-push-to-release", 1458 "beetmover-maven", 1459 ): 1460 worker.setdefault("max-run-time", 600) 1461 elif worker["implementation"] == "push-apk": 1462 worker.setdefault("commit", False) 1463 1464 yield task 1465 1466 1467@transforms.add 1468def task_name_from_label(config, tasks): 1469 for task in tasks: 1470 if "label" not in task: 1471 if "name" not in task: 1472 raise Exception("task has neither a name nor a label") 1473 task["label"] = "{}-{}".format(config.kind, task["name"]) 1474 if task.get("name"): 1475 del task["name"] 1476 yield task 1477 1478 1479UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\ 1480The shipping product {product} is not in the list of configured products in 1481`taskcluster/ci/config.yml'. 1482""" 1483 1484 1485def validate_shipping_product(config, product): 1486 if product not in config.graph_config["release-promotion"]["products"]: 1487 raise Exception(UNSUPPORTED_SHIPPING_PRODUCT_ERROR.format(product=product)) 1488 1489 1490@transforms.add 1491def validate(config, tasks): 1492 for task in tasks: 1493 validate_schema( 1494 task_description_schema, 1495 task, 1496 "In task {!r}:".format(task.get("label", "?no-label?")), 1497 ) 1498 validate_schema( 1499 payload_builders[task["worker"]["implementation"]].schema, 1500 task["worker"], 1501 "In task.run {!r}:".format(task.get("label", "?no-label?")), 1502 ) 1503 if task["shipping-product"] is not None: 1504 validate_shipping_product(config, task["shipping-product"]) 1505 yield task 1506 1507 1508@index_builder("generic") 1509def add_generic_index_routes(config, task): 1510 index = task.get("index") 1511 routes = task.setdefault("routes", []) 1512 1513 verify_index(config, index) 1514 1515 subs = config.params.copy() 1516 subs["job-name"] = index["job-name"] 1517 subs["build_date_long"] = time.strftime( 1518 "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) 1519 ) 1520 subs["build_date"] = time.strftime( 1521 "%Y.%m.%d", time.gmtime(config.params["build_date"]) 1522 ) 1523 subs["product"] = index["product"] 1524 subs["trust-domain"] = config.graph_config["trust-domain"] 1525 subs["branch_rev"] = get_branch_rev(config) 1526 1527 project = config.params.get("project") 1528 1529 for tpl in V2_ROUTE_TEMPLATES: 1530 routes.append(tpl.format(**subs)) 1531 1532 # Additionally alias all tasks for "trunk" repos into a common 1533 # namespace. 1534 if project and project in TRUNK_PROJECTS: 1535 for tpl in V2_TRUNK_ROUTE_TEMPLATES: 1536 routes.append(tpl.format(**subs)) 1537 1538 return task 1539 1540 1541@index_builder("shippable") 1542def add_shippable_index_routes(config, task): 1543 index = task.get("index") 1544 routes = task.setdefault("routes", []) 1545 1546 verify_index(config, index) 1547 1548 subs = config.params.copy() 1549 subs["job-name"] = index["job-name"] 1550 subs["build_date_long"] = time.strftime( 1551 "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) 1552 ) 1553 subs["build_date"] = time.strftime( 1554 "%Y.%m.%d", time.gmtime(config.params["build_date"]) 1555 ) 1556 subs["product"] = index["product"] 1557 subs["trust-domain"] = config.graph_config["trust-domain"] 1558 subs["branch_rev"] = get_branch_rev(config) 1559 1560 for tpl in V2_SHIPPABLE_TEMPLATES: 1561 routes.append(tpl.format(**subs)) 1562 1563 # Also add routes for en-US 1564 task = add_shippable_l10n_index_routes(config, task, force_locale="en-US") 1565 1566 return task 1567 1568 1569@index_builder("shippable-with-multi-l10n") 1570def add_shippable_multi_index_routes(config, task): 1571 task = add_shippable_index_routes(config, task) 1572 task = add_l10n_index_routes(config, task, force_locale="multi") 1573 return task 1574 1575 1576@index_builder("l10n") 1577def add_l10n_index_routes(config, task, force_locale=None): 1578 index = task.get("index") 1579 routes = task.setdefault("routes", []) 1580 1581 verify_index(config, index) 1582 1583 subs = config.params.copy() 1584 subs["job-name"] = index["job-name"] 1585 subs["build_date_long"] = time.strftime( 1586 "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) 1587 ) 1588 subs["product"] = index["product"] 1589 subs["trust-domain"] = config.graph_config["trust-domain"] 1590 subs["branch_rev"] = get_branch_rev(config) 1591 1592 locales = task["attributes"].get( 1593 "chunk_locales", task["attributes"].get("all_locales") 1594 ) 1595 # Some tasks has only one locale set 1596 if task["attributes"].get("locale"): 1597 locales = [task["attributes"]["locale"]] 1598 1599 if force_locale: 1600 # Used for en-US and multi-locale 1601 locales = [force_locale] 1602 1603 if not locales: 1604 raise Exception("Error: Unable to use l10n index for tasks without locales") 1605 1606 # If there are too many locales, we can't write a route for all of them 1607 # See Bug 1323792 1608 if len(locales) > 18: # 18 * 3 = 54, max routes = 64 1609 return task 1610 1611 for locale in locales: 1612 for tpl in V2_L10N_TEMPLATES: 1613 routes.append(tpl.format(locale=locale, **subs)) 1614 1615 return task 1616 1617 1618@index_builder("shippable-l10n") 1619def add_shippable_l10n_index_routes(config, task, force_locale=None): 1620 index = task.get("index") 1621 routes = task.setdefault("routes", []) 1622 1623 verify_index(config, index) 1624 1625 subs = config.params.copy() 1626 subs["job-name"] = index["job-name"] 1627 subs["build_date_long"] = time.strftime( 1628 "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) 1629 ) 1630 subs["product"] = index["product"] 1631 subs["trust-domain"] = config.graph_config["trust-domain"] 1632 subs["branch_rev"] = get_branch_rev(config) 1633 1634 locales = task["attributes"].get( 1635 "chunk_locales", task["attributes"].get("all_locales") 1636 ) 1637 # Some tasks has only one locale set 1638 if task["attributes"].get("locale"): 1639 locales = [task["attributes"]["locale"]] 1640 1641 if force_locale: 1642 # Used for en-US and multi-locale 1643 locales = [force_locale] 1644 1645 if not locales: 1646 raise Exception("Error: Unable to use l10n index for tasks without locales") 1647 1648 # If there are too many locales, we can't write a route for all of them 1649 # See Bug 1323792 1650 if len(locales) > 18: # 18 * 3 = 54, max routes = 64 1651 return task 1652 1653 for locale in locales: 1654 for tpl in V2_SHIPPABLE_L10N_TEMPLATES: 1655 routes.append(tpl.format(locale=locale, **subs)) 1656 1657 return task 1658 1659 1660def add_geckoview_index_routes(config, task): 1661 index = task.get("index") 1662 routes = task.setdefault("routes", []) 1663 geckoview_version = _compute_geckoview_version( 1664 config.params["app_version"], config.params["moz_build_date"] 1665 ) 1666 1667 subs = { 1668 "geckoview-version": geckoview_version, 1669 "job-name": index["job-name"], 1670 "product": index["product"], 1671 "project": config.params["project"], 1672 "trust-domain": config.graph_config["trust-domain"], 1673 } 1674 routes.append(V2_GECKOVIEW_RELEASE.format(**subs)) 1675 1676 return task 1677 1678 1679@index_builder("android-shippable") 1680def add_android_shippable_index_routes(config, task): 1681 task = add_shippable_index_routes(config, task) 1682 task = add_geckoview_index_routes(config, task) 1683 1684 return task 1685 1686 1687@index_builder("android-shippable-with-multi-l10n") 1688def add_android_shippable_multi_index_routes(config, task): 1689 task = add_shippable_multi_index_routes(config, task) 1690 task = add_geckoview_index_routes(config, task) 1691 1692 return task 1693 1694 1695@transforms.add 1696def add_index_routes(config, tasks): 1697 for task in tasks: 1698 index = task.get("index", {}) 1699 1700 # The default behavior is to rank tasks according to their tier 1701 extra_index = task.setdefault("extra", {}).setdefault("index", {}) 1702 rank = index.get("rank", "by-tier") 1703 1704 if rank == "by-tier": 1705 # rank is zero for non-tier-1 tasks and based on pushid for others; 1706 # this sorts tier-{2,3} builds below tier-1 in the index 1707 tier = task.get("treeherder", {}).get("tier", 3) 1708 extra_index["rank"] = 0 if tier > 1 else int(config.params["build_date"]) 1709 elif rank == "build_date": 1710 extra_index["rank"] = int(config.params["build_date"]) 1711 else: 1712 extra_index["rank"] = rank 1713 1714 if not index: 1715 yield task 1716 continue 1717 1718 index_type = index.get("type", "generic") 1719 task = index_builders[index_type](config, task) 1720 1721 del task["index"] 1722 yield task 1723 1724 1725@transforms.add 1726def try_task_config_env(config, tasks): 1727 """Set environment variables in the task.""" 1728 env = config.params["try_task_config"].get("env") 1729 # Find all implementations that have an 'env' key. 1730 implementations = { 1731 name 1732 for name, builder in payload_builders.items() 1733 if "env" in builder.schema.schema 1734 } 1735 for task in tasks: 1736 if env and task["worker"]["implementation"] in implementations: 1737 task["worker"]["env"].update(env) 1738 yield task 1739 1740 1741@transforms.add 1742def try_task_config_chemspill_prio(config, tasks): 1743 """Increase the priority from lowest and very-low -> low, but leave others unchanged.""" 1744 chemspill_prio = config.params["try_task_config"].get("chemspill-prio") 1745 for task in tasks: 1746 if chemspill_prio and task["priority"] in ("lowest", "very-low"): 1747 task["priority"] = "low" 1748 yield task 1749 1750 1751@transforms.add 1752def try_task_config_routes(config, tasks): 1753 """Set routes in the task.""" 1754 routes = config.params["try_task_config"].get("routes") 1755 for task in tasks: 1756 if routes: 1757 task_routes = task.setdefault("routes", []) 1758 task_routes.extend(routes) 1759 yield task 1760 1761 1762@transforms.add 1763def build_task(config, tasks): 1764 for task in tasks: 1765 level = str(config.params["level"]) 1766 1767 if task["worker-type"] in config.params["try_task_config"].get( 1768 "worker-overrides", {} 1769 ): 1770 worker_pool = config.params["try_task_config"]["worker-overrides"][ 1771 task["worker-type"] 1772 ] 1773 provisioner_id, worker_type = worker_pool.split("/", 1) 1774 else: 1775 provisioner_id, worker_type = get_worker_type( 1776 config.graph_config, 1777 task["worker-type"], 1778 level=level, 1779 release_level=config.params.release_level(), 1780 ) 1781 task["worker-type"] = "/".join([provisioner_id, worker_type]) 1782 project = config.params["project"] 1783 1784 routes = task.get("routes", []) 1785 scopes = [ 1786 s.format(level=level, project=project) for s in task.get("scopes", []) 1787 ] 1788 1789 # set up extra 1790 extra = task.get("extra", {}) 1791 extra["parent"] = {"task-reference": "<decision>"} 1792 task_th = task.get("treeherder") 1793 if task_th: 1794 extra.setdefault("treeherder-platform", task_th["platform"]) 1795 treeherder = extra.setdefault("treeherder", {}) 1796 1797 machine_platform, collection = task_th["platform"].split("/", 1) 1798 treeherder["machine"] = {"platform": machine_platform} 1799 treeherder["collection"] = {collection: True} 1800 1801 group_names = config.graph_config["treeherder"]["group-names"] 1802 groupSymbol, symbol = split_symbol(task_th["symbol"]) 1803 if groupSymbol != "?": 1804 treeherder["groupSymbol"] = groupSymbol 1805 if groupSymbol not in group_names: 1806 path = os.path.join(config.path, task.get("job-from", "")) 1807 raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol, path)) 1808 treeherder["groupName"] = group_names[groupSymbol] 1809 treeherder["symbol"] = symbol 1810 if len(symbol) > 25 or len(groupSymbol) > 25: 1811 raise RuntimeError( 1812 "Treeherder group and symbol names must not be longer than " 1813 "25 characters: {} (see {})".format( 1814 task_th["symbol"], 1815 TC_TREEHERDER_SCHEMA_URL, 1816 ) 1817 ) 1818 treeherder["jobKind"] = task_th["kind"] 1819 treeherder["tier"] = task_th["tier"] 1820 1821 branch_rev = get_branch_rev(config) 1822 1823 routes.append( 1824 "{}.v2.{}.{}".format( 1825 TREEHERDER_ROUTE_ROOT, 1826 config.params["project"], 1827 branch_rev, 1828 ) 1829 ) 1830 1831 if "expires-after" in task: 1832 if config.params.is_try(): 1833 delta = value_of(task["expires-after"]) 1834 if delta.days >= 28: 1835 task["expires-after"] = "28 days" 1836 else: 1837 task["expires-after"] = "28 days" if config.params.is_try() else "1 year" 1838 1839 if "deadline-after" not in task: 1840 task["deadline-after"] = "1 day" 1841 1842 if "priority" not in task: 1843 task["priority"] = get_default_priority( 1844 config.graph_config, config.params["project"] 1845 ) 1846 1847 tags = task.get("tags", {}) 1848 attributes = task.get("attributes", {}) 1849 1850 tags.update( 1851 { 1852 "createdForUser": config.params["owner"], 1853 "kind": config.kind, 1854 "label": task["label"], 1855 "retrigger": "true" if attributes.get("retrigger", False) else "false", 1856 } 1857 ) 1858 1859 task_def = { 1860 "provisionerId": provisioner_id, 1861 "workerType": worker_type, 1862 "routes": routes, 1863 "created": {"relative-datestamp": "0 seconds"}, 1864 "deadline": {"relative-datestamp": task["deadline-after"]}, 1865 "expires": {"relative-datestamp": task["expires-after"]}, 1866 "scopes": scopes, 1867 "metadata": { 1868 "description": task["description"], 1869 "name": task["label"], 1870 "owner": config.params["owner"], 1871 "source": config.params.file_url(config.path, pretty=True), 1872 }, 1873 "extra": extra, 1874 "tags": tags, 1875 "priority": task["priority"], 1876 } 1877 1878 if task.get("requires", None): 1879 task_def["requires"] = task["requires"] 1880 1881 if task_th: 1882 # link back to treeherder in description 1883 th_job_link = ( 1884 "https://treeherder.mozilla.org/#/jobs?repo={}&revision={}&selectedTaskRun=<self>" 1885 ).format(config.params["project"], branch_rev) 1886 task_def["metadata"]["description"] = { 1887 "task-reference": "{description} ([Treeherder job]({th_job_link}))".format( 1888 description=task_def["metadata"]["description"], 1889 th_job_link=th_job_link, 1890 ) 1891 } 1892 1893 # add the payload and adjust anything else as required (e.g., scopes) 1894 payload_builders[task["worker"]["implementation"]].builder( 1895 config, task, task_def 1896 ) 1897 1898 # Resolve run-on-projects 1899 build_platform = attributes.get("build_platform") 1900 resolve_keyed_by( 1901 task, 1902 "run-on-projects", 1903 item_name=task["label"], 1904 **{"build-platform": build_platform} 1905 ) 1906 attributes["run_on_projects"] = task.get("run-on-projects", ["all"]) 1907 attributes["always_target"] = task["always-target"] 1908 # This logic is here since downstream tasks don't always match their 1909 # upstream dependency's shipping_phase. 1910 # A text_type task['shipping-phase'] takes precedence, then 1911 # an existing attributes['shipping_phase'], then fall back to None. 1912 if task.get("shipping-phase") is not None: 1913 attributes["shipping_phase"] = task["shipping-phase"] 1914 else: 1915 attributes.setdefault("shipping_phase", None) 1916 # shipping_product will always match the upstream task's 1917 # shipping_product, so a pre-set existing attributes['shipping_product'] 1918 # takes precedence over task['shipping-product']. However, make sure 1919 # we don't have conflicting values. 1920 if task.get("shipping-product") and attributes.get("shipping_product") not in ( 1921 None, 1922 task["shipping-product"], 1923 ): 1924 raise Exception( 1925 "{} shipping_product {} doesn't match task shipping-product {}!".format( 1926 task["label"], 1927 attributes["shipping_product"], 1928 task["shipping-product"], 1929 ) 1930 ) 1931 attributes.setdefault("shipping_product", task["shipping-product"]) 1932 1933 # Set MOZ_AUTOMATION on all jobs. 1934 if task["worker"]["implementation"] in ( 1935 "generic-worker", 1936 "docker-worker", 1937 ): 1938 payload = task_def.get("payload") 1939 if payload: 1940 env = payload.setdefault("env", {}) 1941 env["MOZ_AUTOMATION"] = "1" 1942 1943 dependencies = task.get("dependencies", {}) 1944 if_dependencies = task.get("if-dependencies", []) 1945 if if_dependencies: 1946 for i, dep in enumerate(if_dependencies): 1947 if dep in dependencies: 1948 if_dependencies[i] = dependencies[dep] 1949 continue 1950 1951 raise Exception( 1952 "{label} specifies '{dep}' in if-dependencies, " 1953 "but {dep} is not a dependency!".format( 1954 label=task["label"], dep=dep 1955 ) 1956 ) 1957 1958 yield { 1959 "label": task["label"], 1960 "description": task["description"], 1961 "task": task_def, 1962 "dependencies": dependencies, 1963 "if-dependencies": if_dependencies, 1964 "soft-dependencies": task.get("soft-dependencies", []), 1965 "attributes": attributes, 1966 "optimization": task.get("optimization", None), 1967 "release-artifacts": task.get("release-artifacts", []), 1968 } 1969 1970 1971@transforms.add 1972def chain_of_trust(config, tasks): 1973 for task in tasks: 1974 if task["task"].get("payload", {}).get("features", {}).get("chainOfTrust"): 1975 image = task.get("dependencies", {}).get("docker-image") 1976 if image: 1977 cot = ( 1978 task["task"].setdefault("extra", {}).setdefault("chainOfTrust", {}) 1979 ) 1980 cot.setdefault("inputs", {})["docker-image"] = { 1981 "task-reference": "<docker-image>" 1982 } 1983 yield task 1984 1985 1986@transforms.add 1987def check_task_identifiers(config, tasks): 1988 """Ensures that all tasks have well defined identifiers: 1989 ^[a-zA-Z0-9_-]{1,38}$ 1990 """ 1991 e = re.compile("^[a-zA-Z0-9_-]{1,38}$") 1992 for task in tasks: 1993 for attrib in ("workerType", "provisionerId"): 1994 if not e.match(task["task"][attrib]): 1995 raise Exception( 1996 "task {}.{} is not a valid identifier: {}".format( 1997 task["label"], attrib, task["task"][attrib] 1998 ) 1999 ) 2000 yield task 2001 2002 2003@transforms.add 2004def check_task_dependencies(config, tasks): 2005 """Ensures that tasks don't have more than 100 dependencies.""" 2006 for task in tasks: 2007 if len(task["dependencies"]) > MAX_DEPENDENCIES: 2008 raise Exception( 2009 "task {}/{} has too many dependencies ({} > {})".format( 2010 config.kind, 2011 task["label"], 2012 len(task["dependencies"]), 2013 MAX_DEPENDENCIES, 2014 ) 2015 ) 2016 yield task 2017 2018 2019def check_caches_are_volumes(task): 2020 """Ensures that all cache paths are defined as volumes. 2021 2022 Caches and volumes are the only filesystem locations whose content 2023 isn't defined by the Docker image itself. Some caches are optional 2024 depending on the job environment. We want paths that are potentially 2025 caches to have as similar behavior regardless of whether a cache is 2026 used. To help enforce this, we require that all paths used as caches 2027 to be declared as Docker volumes. This check won't catch all offenders. 2028 But it is better than nothing. 2029 """ 2030 volumes = set(six.ensure_text(s) for s in task["worker"]["volumes"]) 2031 paths = set( 2032 six.ensure_text(c["mount-point"]) for c in task["worker"].get("caches", []) 2033 ) 2034 missing = paths - volumes 2035 2036 if not missing: 2037 return 2038 2039 raise Exception( 2040 "task %s (image %s) has caches that are not declared as " 2041 "Docker volumes: %s " 2042 "(have you added them as VOLUMEs in the Dockerfile?)" 2043 % (task["label"], task["worker"]["docker-image"], ", ".join(sorted(missing))) 2044 ) 2045 2046 2047def check_required_volumes(task): 2048 """ 2049 Ensures that all paths that are required to be volumes are defined as volumes. 2050 2051 Performance of writing to files in poor in directories not marked as 2052 volumes, in docker. Ensure that paths that are often written to are marked 2053 as volumes. 2054 """ 2055 volumes = set(task["worker"]["volumes"]) 2056 paths = set(task["worker"].get("required-volumes", [])) 2057 missing = paths - volumes 2058 2059 if not missing: 2060 return 2061 2062 raise Exception( 2063 "task %s (image %s) has paths that should be volumes for peformance " 2064 "that are not declared as Docker volumes: %s " 2065 "(have you added them as VOLUMEs in the Dockerfile?)" 2066 % (task["label"], task["worker"]["docker-image"], ", ".join(sorted(missing))) 2067 ) 2068 2069 2070@transforms.add 2071def check_run_task_caches(config, tasks): 2072 """Audit for caches requiring run-task. 2073 2074 run-task manages caches in certain ways. If a cache managed by run-task 2075 is used by a non run-task task, it could cause problems. So we audit for 2076 that and make sure certain cache names are exclusive to run-task. 2077 2078 IF YOU ARE TEMPTED TO MAKE EXCLUSIONS TO THIS POLICY, YOU ARE LIKELY 2079 CONTRIBUTING TECHNICAL DEBT AND WILL HAVE TO SOLVE MANY OF THE PROBLEMS 2080 THAT RUN-TASK ALREADY SOLVES. THINK LONG AND HARD BEFORE DOING THAT. 2081 """ 2082 re_reserved_caches = re.compile( 2083 """^ 2084 (checkouts|tooltool-cache) 2085 """, 2086 re.VERBOSE, 2087 ) 2088 2089 re_sparse_checkout_cache = re.compile("^checkouts-sparse") 2090 2091 cache_prefix = "{trust_domain}-level-{level}-".format( 2092 trust_domain=config.graph_config["trust-domain"], 2093 level=config.params["level"], 2094 ) 2095 2096 suffix = _run_task_suffix() 2097 2098 for task in tasks: 2099 payload = task["task"].get("payload", {}) 2100 command = payload.get("command") or [""] 2101 2102 main_command = command[0] if isinstance(command[0], text_type) else "" 2103 run_task = main_command.endswith("run-task") 2104 2105 require_sparse_cache = False 2106 have_sparse_cache = False 2107 2108 if run_task: 2109 for arg in command[1:]: 2110 if not isinstance(arg, text_type): 2111 continue 2112 2113 if arg == "--": 2114 break 2115 2116 if arg.startswith("--gecko-sparse-profile"): 2117 if "=" not in arg: 2118 raise Exception( 2119 "{} is specifying `--gecko-sparse-profile` to run-task " 2120 "as two arguments. Unable to determine if the sparse " 2121 "profile exists.".format(task["label"]) 2122 ) 2123 _, sparse_profile = arg.split("=", 1) 2124 if not os.path.exists(os.path.join(GECKO, sparse_profile)): 2125 raise Exception( 2126 "{} is using non-existant sparse profile {}.".format( 2127 task["label"], sparse_profile 2128 ) 2129 ) 2130 require_sparse_cache = True 2131 break 2132 2133 for cache in payload.get("cache", {}): 2134 if not cache.startswith(cache_prefix): 2135 raise Exception( 2136 "{} is using a cache ({}) which is not appropriate " 2137 "for its trust-domain and level. It should start with {}.".format( 2138 task["label"], cache, cache_prefix 2139 ) 2140 ) 2141 2142 cache = cache[len(cache_prefix) :] 2143 2144 if re_sparse_checkout_cache.match(cache): 2145 have_sparse_cache = True 2146 2147 if not re_reserved_caches.match(cache): 2148 continue 2149 2150 if not run_task: 2151 raise Exception( 2152 "%s is using a cache (%s) reserved for run-task " 2153 "change the task to use run-task or use a different " 2154 "cache name" % (task["label"], cache) 2155 ) 2156 2157 if not cache.endswith(suffix): 2158 raise Exception( 2159 "%s is using a cache (%s) reserved for run-task " 2160 "but the cache name is not dependent on the contents " 2161 "of run-task; change the cache name to conform to the " 2162 "naming requirements" % (task["label"], cache) 2163 ) 2164 2165 if require_sparse_cache and not have_sparse_cache: 2166 raise Exception( 2167 "%s is using a sparse checkout but not using " 2168 "a sparse checkout cache; change the checkout " 2169 "cache name so it is sparse aware" % task["label"] 2170 ) 2171 2172 yield task 2173