1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4""" 5These transforms construct a task description to run the given test, based on a 6test description. The implementation here is shared among all test kinds, but 7contains specific support for how we run tests in Gecko (via mozharness, 8invoked in particular ways). 9 10This is a good place to translate a test-description option such as 11`single-core: true` to the implementation of that option in a task description 12(worker options, mozharness commandline, environment variables, etc.) 13 14The test description should be fully formed by the time it reaches these 15transforms, and these transforms should not embody any specific knowledge about 16what should run where. this is the wrong place for special-casing platforms, 17for example - use `all_tests.py` instead. 18""" 19 20from __future__ import absolute_import, print_function, unicode_literals 21 22import copy 23import logging 24from six import string_types, text_type 25 26from mozbuild.schedules import INCLUSIVE_COMPONENTS 27from moztest.resolve import TEST_SUITES 28from voluptuous import ( 29 Any, 30 Optional, 31 Required, 32 Exclusive, 33) 34 35import taskgraph 36from taskgraph.transforms.base import TransformSequence 37from taskgraph.util.attributes import match_run_on_projects, keymatch 38from taskgraph.util.keyed_by import evaluate_keyed_by 39from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema 40from taskgraph.util.templates import merge 41from taskgraph.util.treeherder import split_symbol, join_symbol, add_suffix 42from taskgraph.util.platforms import platform_family 43from taskgraph.util.schema import ( 44 optionally_keyed_by, 45 Schema, 46) 47from taskgraph.util.chunking import ( 48 chunk_manifests, 49 get_runtimes, 50 guess_mozinfo_from_task, 51 manifest_loaders, 52) 53from taskgraph.util.taskcluster import ( 54 get_artifact_path, 55 get_index_url, 56) 57from taskgraph.util.perfile import perfile_number_of_chunks 58 59 60# default worker types keyed by instance-size 61LINUX_WORKER_TYPES = { 62 'large': 't-linux-large', 63 'xlarge': 't-linux-xlarge', 64 'default': 't-linux-large', 65} 66 67# windows worker types keyed by test-platform and virtualization 68WINDOWS_WORKER_TYPES = { 69 'windows7-32': { 70 'virtual': 't-win7-32', 71 'virtual-with-gpu': 't-win7-32-gpu', 72 'hardware': 't-win10-64-hw', 73 }, 74 'windows7-32-shippable': { 75 'virtual': 't-win7-32', 76 'virtual-with-gpu': 't-win7-32-gpu', 77 'hardware': 't-win10-64-hw', 78 }, 79 'windows7-32-devedition': { 80 'virtual': 't-win7-32', 81 'virtual-with-gpu': 't-win7-32-gpu', 82 'hardware': 't-win10-64-hw', 83 }, 84 'windows7-32-mingwclang': { 85 'virtual': 't-win7-32', 86 'virtual-with-gpu': 't-win7-32-gpu', 87 'hardware': 't-win10-64-hw', 88 }, 89 'windows10-64': { 90 'virtual': 't-win10-64', 91 'virtual-with-gpu': 't-win10-64-gpu-s', 92 'hardware': 't-win10-64-hw', 93 }, 94 'windows10-aarch64': { 95 'virtual': 't-win64-aarch64-laptop', 96 'virtual-with-gpu': 't-win64-aarch64-laptop', 97 'hardware': 't-win64-aarch64-laptop', 98 }, 99 'windows10-64-ccov': { 100 'virtual': 't-win10-64', 101 'virtual-with-gpu': 't-win10-64-gpu-s', 102 'hardware': 't-win10-64-hw', 103 }, 104 'windows10-64-devedition': { 105 'virtual': 't-win10-64', 106 'virtual-with-gpu': 't-win10-64-gpu-s', 107 'hardware': 't-win10-64-hw', 108 }, 109 'windows10-64-shippable': { 110 'virtual': 't-win10-64', 111 'virtual-with-gpu': 't-win10-64-gpu-s', 112 'hardware': 't-win10-64-hw', 113 }, 114 'windows10-64-asan': { 115 'virtual': 't-win10-64', 116 'virtual-with-gpu': 't-win10-64-gpu-s', 117 'hardware': 't-win10-64-hw', 118 }, 119 'windows10-64-qr': { 120 'virtual': 't-win10-64', 121 'virtual-with-gpu': 't-win10-64-gpu-s', 122 'hardware': 't-win10-64-hw', 123 }, 124 'windows10-64-shippable-qr': { 125 'virtual': 't-win10-64', 126 'virtual-with-gpu': 't-win10-64-gpu-s', 127 'hardware': 't-win10-64-hw', 128 }, 129 'windows10-64-mingwclang': { 130 'virtual': 't-win10-64', 131 'virtual-with-gpu': 't-win10-64-gpu-s', 132 'hardware': 't-win10-64-hw', 133 }, 134 'windows10-64-ref-hw-2017': { 135 'virtual': 't-win10-64', 136 'virtual-with-gpu': 't-win10-64-gpu-s', 137 'hardware': 't-win10-64-ref-hw', 138 }, 139} 140 141# os x worker types keyed by test-platform 142MACOSX_WORKER_TYPES = { 143 'macosx1014-64': 't-osx-1014', 144 'macosx1014-64-power': 't-osx-1014-power' 145} 146 147 148def runs_on_central(task): 149 return match_run_on_projects('mozilla-central', task['run-on-projects']) 150 151 152def gv_e10s_multi_filter(task): 153 return ( 154 get_mobile_project(task) == 'geckoview' and 155 task['e10s'] 156 ) 157 158 159def fission_filter(task): 160 return ( 161 runs_on_central(task) and 162 task.get('e10s') in (True, 'both') and 163 get_mobile_project(task) != 'fennec' 164 ) 165 166 167TEST_VARIANTS = { 168 'geckoview-e10s-multi': { 169 'description': "{description} with e10s-multi enabled", 170 'filterfn': gv_e10s_multi_filter, 171 'replace': { 172 'run-on-projects': ['trunk'], 173 }, 174 'suffix': 'e10s-multi', 175 'merge': { 176 'mozharness': { 177 'extra-options': [ 178 '--setpref=dom.ipc.processCount=3', 179 ], 180 }, 181 }, 182 }, 183 'fission': { 184 'description': "{description} with fission enabled", 185 'filterfn': fission_filter, 186 'suffix': 'fis', 187 'replace': { 188 'e10s': True, 189 }, 190 'merge': { 191 # Ensures the default state is to not run anywhere. 192 'fission-run-on-projects': [], 193 'mozharness': { 194 'extra-options': ['--setpref=fission.autostart=true', 195 '--setpref=dom.serviceWorkers.parent_intercept=true', 196 '--setpref=browser.tabs.documentchannel=true'], 197 }, 198 }, 199 }, 200 'socketprocess': { 201 'description': "{description} with socket process enabled", 202 'suffix': 'spi', 203 'merge': { 204 'mozharness': { 205 'extra-options': [ 206 '--setpref=media.peerconnection.mtransport_process=true', 207 '--setpref=network.process.enabled=true', 208 ], 209 } 210 } 211 }, 212 'socketprocess_networking': { 213 'description': "{description} with networking on socket process enabled", 214 'suffix': 'spi-nw', 215 'merge': { 216 'mozharness': { 217 'extra-options': [ 218 '--setpref=network.process.enabled=true', 219 '--setpref=network.http.network_access_on_socket_process.enabled=true', 220 '--setpref=network.ssl_tokens_cache_enabled=true', 221 ], 222 } 223 } 224 } 225} 226 227 228CHUNK_SUITES_BLACKLIST = ( 229 'awsy', 230 'cppunittest', 231 'crashtest', 232 'crashtest-qr', 233 'firefox-ui-functional-local', 234 'firefox-ui-functional-remote', 235 'geckoview-junit', 236 'gtest', 237 'jittest', 238 'jsreftest', 239 'marionette', 240 'mochitest-browser-chrome-screenshots', 241 'mochitest-browser-chrome-thunderbird', 242 'mochitest-valgrind-plain', 243 'mochitest-webgl1-core', 244 'mochitest-webgl1-ext', 245 'mochitest-webgl2-core', 246 'mochitest-webgl2-ext', 247 'raptor', 248 'reftest', 249 'reftest-qr', 250 'reftest-gpu', 251 'reftest-no-accel', 252 'talos', 253 'telemetry-tests-client', 254 'test-coverage', 255 'test-coverage-wpt', 256 'test-verify', 257 'test-verify-gpu', 258 'test-verify-wpt', 259 'web-platform-tests-backlog', 260 'web-platform-tests-crashtest', 261 'web-platform-tests-reftest', 262 'web-platform-tests-reftest-backlog', 263 'web-platform-tests-wdspec', 264) 265"""These suites will be chunked at test runtime rather than here in the taskgraph.""" 266 267 268DYNAMIC_CHUNK_DURATION = 20 * 60 # seconds 269"""The approximate time each test chunk should take to run.""" 270 271 272logger = logging.getLogger(__name__) 273 274transforms = TransformSequence() 275 276# Schema for a test description 277# 278# *****WARNING***** 279# 280# This is a great place for baffling cruft to accumulate, and that makes 281# everyone move more slowly. Be considerate of your fellow hackers! 282# See the warnings in taskcluster/docs/how-tos.rst 283# 284# *****WARNING***** 285test_description_schema = Schema({ 286 # description of the suite, for the task metadata 287 'description': text_type, 288 289 # test suite category and name 290 Optional('suite'): Any( 291 text_type, 292 {Optional('category'): text_type, Optional('name'): text_type}, 293 ), 294 295 # base work directory used to set up the task. 296 Optional('workdir'): optionally_keyed_by( 297 'test-platform', 298 Any(text_type, 'default')), 299 300 # the name by which this test suite is addressed in try syntax; defaults to 301 # the test-name. This will translate to the `unittest_try_name` or 302 # `talos_try_name` attribute. 303 Optional('try-name'): text_type, 304 305 # additional tags to mark up this type of test 306 Optional('tags'): {text_type: object}, 307 308 # the symbol, or group(symbol), under which this task should appear in 309 # treeherder. 310 'treeherder-symbol': text_type, 311 312 # the value to place in task.extra.treeherder.machine.platform; ideally 313 # this is the same as build-platform, and that is the default, but in 314 # practice it's not always a match. 315 Optional('treeherder-machine-platform'): text_type, 316 317 # attributes to appear in the resulting task (later transforms will add the 318 # common attributes) 319 Optional('attributes'): {text_type: object}, 320 321 # relative path (from config.path) to the file task was defined in 322 Optional('job-from'): text_type, 323 324 # The `run_on_projects` attribute, defaulting to "all". This dictates the 325 # projects on which this task should be included in the target task set. 326 # See the attributes documentation for details. 327 # 328 # Note that the special case 'built-projects', the default, uses the parent 329 # build task's run-on-projects, meaning that tests run only on platforms 330 # that are built. 331 Optional('run-on-projects'): optionally_keyed_by( 332 'test-platform', 333 'test-name', 334 Any([text_type], 'built-projects')), 335 336 # Same as `run-on-projects` except it only applies to Fission tasks. Fission 337 # tasks will ignore `run_on_projects` and non-Fission tasks will ignore 338 # `fission-run-on-projects`. 339 Optional('fission-run-on-projects'): optionally_keyed_by( 340 'test-platform', 341 Any([text_type], 'built-projects')), 342 343 # the sheriffing tier for this task (default: set based on test platform) 344 Optional('tier'): optionally_keyed_by( 345 'test-platform', 346 Any(int, 'default')), 347 348 # Same as `tier` except it only applies to Fission tasks. Fission tasks 349 # will ignore `tier` and non-Fission tasks will ignore `fission-tier`. 350 Optional('fission-tier'): optionally_keyed_by( 351 'test-platform', 352 Any(int, 'default')), 353 354 # number of chunks to create for this task. This can be keyed by test 355 # platform by passing a dictionary in the `by-test-platform` key. If the 356 # test platform is not found, the key 'default' will be tried. 357 Required('chunks'): optionally_keyed_by( 358 'test-platform', 359 Any(int, 'dynamic')), 360 361 # the time (with unit) after which this task is deleted; default depends on 362 # the branch (see below) 363 Optional('expires-after'): text_type, 364 365 # The different configurations that should be run against this task, defined 366 # in the TEST_VARIANTS object. 367 Optional('variants'): optionally_keyed_by( 368 'test-platform', 'project', 369 Any(list(TEST_VARIANTS))), 370 371 # Whether to run this task with e10s. If false, run 372 # without e10s; if true, run with e10s; if 'both', run one task with and 373 # one task without e10s. E10s tasks have "-e10s" appended to the test name 374 # and treeherder group. 375 Required('e10s'): optionally_keyed_by( 376 'test-platform', 'project', 377 Any(bool, 'both')), 378 379 # Whether the task should run with WebRender enabled or not. 380 Optional('webrender'): bool, 381 382 # The EC2 instance size to run these tests on. 383 Required('instance-size'): optionally_keyed_by( 384 'test-platform', 385 Any('default', 'large', 'xlarge')), 386 387 # type of virtualization or hardware required by test. 388 Required('virtualization'): optionally_keyed_by( 389 'test-platform', 390 Any('virtual', 'virtual-with-gpu', 'hardware')), 391 392 # Whether the task requires loopback audio or video (whatever that may mean 393 # on the platform) 394 Required('loopback-audio'): bool, 395 Required('loopback-video'): bool, 396 397 # Whether the test can run using a software GL implementation on Linux 398 # using the GL compositor. May not be used with "legacy" sized instances 399 # due to poor LLVMPipe performance (bug 1296086). Defaults to true for 400 # unit tests on linux platforms and false otherwise 401 Optional('allow-software-gl-layers'): bool, 402 403 # For tasks that will run in docker-worker, this is the 404 # name of the docker image or in-tree docker image to run the task in. If 405 # in-tree, then a dependency will be created automatically. This is 406 # generally `desktop-test`, or an image that acts an awful lot like it. 407 Required('docker-image'): optionally_keyed_by( 408 'test-platform', 409 Any( 410 # a raw Docker image path (repo/image:tag) 411 text_type, 412 # an in-tree generated docker image (from `taskcluster/docker/<name>`) 413 {'in-tree': text_type}, 414 # an indexed docker image 415 {'indexed': text_type}, 416 ) 417 ), 418 419 # seconds of runtime after which the task will be killed. Like 'chunks', 420 # this can be keyed by test pltaform. 421 Required('max-run-time'): optionally_keyed_by( 422 'test-platform', 423 int), 424 425 # the exit status code that indicates the task should be retried 426 Optional('retry-exit-status'): [int], 427 428 # Whether to perform a gecko checkout. 429 Required('checkout'): bool, 430 431 # Wheter to perform a machine reboot after test is done 432 Optional('reboot'): 433 Any(False, 'always', 'on-exception', 'on-failure'), 434 435 # What to run 436 Required('mozharness'): { 437 # the mozharness script used to run this task 438 Required('script'): optionally_keyed_by( 439 'test-platform', 440 text_type), 441 442 # the config files required for the task 443 Required('config'): optionally_keyed_by( 444 'test-platform', 445 [text_type]), 446 447 # mochitest flavor for mochitest runs 448 Optional('mochitest-flavor'): text_type, 449 450 # any additional actions to pass to the mozharness command 451 Optional('actions'): [text_type], 452 453 # additional command-line options for mozharness, beyond those 454 # automatically added 455 Required('extra-options'): optionally_keyed_by( 456 'test-platform', 457 [text_type]), 458 459 # the artifact name (including path) to test on the build task; this is 460 # generally set in a per-kind transformation 461 Optional('build-artifact-name'): text_type, 462 Optional('installer-url'): text_type, 463 464 # If not false, tooltool downloads will be enabled via relengAPIProxy 465 # for either just public files, or all files. Not supported on Windows 466 Required('tooltool-downloads'): Any( 467 False, 468 'public', 469 'internal', 470 ), 471 472 # Add --blob-upload-branch=<project> mozharness parameter 473 Optional('include-blob-upload-branch'): bool, 474 475 # The setting for --download-symbols (if omitted, the option will not 476 # be passed to mozharness) 477 Optional('download-symbols'): Any(True, 'ondemand'), 478 479 # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the 480 # environment. This is more than just a helpful path setting -- it 481 # causes xpcshell tests to start additional servers, and runs 482 # additional tests. 483 Required('set-moz-node-path'): bool, 484 485 # If true, include chunking information in the command even if the number 486 # of chunks is 1 487 Required('chunked'): optionally_keyed_by( 488 'test-platform', 489 bool), 490 491 Required('requires-signed-builds'): optionally_keyed_by( 492 'test-platform', 493 bool), 494 }, 495 496 # The set of test manifests to run. 497 Optional('test-manifests'): Any( 498 [text_type], 499 {'active': [text_type], 'skipped': [text_type]}, 500 ), 501 502 # The current chunk (if chunking is enabled). 503 Optional('this-chunk'): int, 504 505 # os user groups for test task workers; required scopes, will be 506 # added automatically 507 Optional('os-groups'): optionally_keyed_by( 508 'test-platform', 509 [text_type]), 510 511 Optional('run-as-administrator'): optionally_keyed_by( 512 'test-platform', 513 bool), 514 515 # -- values supplied by the task-generation infrastructure 516 517 # the platform of the build this task is testing 518 'build-platform': text_type, 519 520 # the label of the build task generating the materials to test 521 'build-label': text_type, 522 523 # the label of the signing task generating the materials to test. 524 # Signed builds are used in xpcshell tests on Windows, for instance. 525 Optional('build-signing-label'): text_type, 526 527 # the build's attributes 528 'build-attributes': {text_type: object}, 529 530 # the platform on which the tests will run 531 'test-platform': text_type, 532 533 # limit the test-platforms (as defined in test-platforms.yml) 534 # that the test will run on 535 Optional('limit-platforms'): optionally_keyed_by( 536 'app', 537 [text_type] 538 ), 539 540 # the name of the test (the key in tests.yml) 541 'test-name': text_type, 542 543 # the product name, defaults to firefox 544 Optional('product'): text_type, 545 546 # conditional files to determine when these tests should be run 547 Exclusive(Optional('when'), 'optimization'): { 548 Optional('files-changed'): [text_type], 549 }, 550 551 # Optimization to perform on this task during the optimization phase. 552 # Optimizations are defined in taskcluster/taskgraph/optimize.py. 553 Exclusive(Optional('optimization'), 'optimization'): OptimizationSchema, 554 555 # The SCHEDULES component for this task; this defaults to the suite 556 # (not including the flavor) but can be overridden here. 557 Exclusive(Optional('schedules-component'), 'optimization'): Any( 558 text_type, 559 [text_type], 560 ), 561 562 Optional('worker-type'): optionally_keyed_by( 563 'test-platform', 564 Any(text_type, None), 565 ), 566 567 Optional( 568 'require-signed-extensions', 569 description="Whether the build being tested requires extensions be signed.", 570 ): optionally_keyed_by('release-type', 'test-platform', bool), 571 572 # The target name, specifying the build artifact to be tested. 573 # If None or not specified, a transform sets the target based on OS: 574 # target.dmg (Mac), target.apk (Android), target.tar.bz2 (Linux), 575 # or target.zip (Windows). 576 Optional('target'): optionally_keyed_by( 577 'test-platform', 578 Any(text_type, None, {'index': text_type, 'name': text_type}), 579 ), 580 581 # A list of artifacts to install from 'fetch' tasks. 582 Optional('fetches'): { 583 text_type: optionally_keyed_by('test-platform', [text_type]) 584 }, 585}, required=True) 586 587 588@transforms.add 589def handle_keyed_by_mozharness(config, tasks): 590 """Resolve a mozharness field if it is keyed by something""" 591 fields = [ 592 'mozharness', 593 'mozharness.chunked', 594 'mozharness.config', 595 'mozharness.extra-options', 596 'mozharness.requires-signed-builds', 597 'mozharness.script', 598 ] 599 for task in tasks: 600 for field in fields: 601 resolve_keyed_by(task, field, item_name=task['test-name']) 602 yield task 603 604 605@transforms.add 606def set_defaults(config, tasks): 607 for task in tasks: 608 build_platform = task['build-platform'] 609 if build_platform.startswith('android'): 610 # all Android test tasks download internal objects from tooltool 611 task['mozharness']['tooltool-downloads'] = 'internal' 612 task['mozharness']['actions'] = ['get-secrets'] 613 614 # loopback-video is always true for Android, but false for other 615 # platform phyla 616 task['loopback-video'] = True 617 task['mozharness']['set-moz-node-path'] = True 618 619 # software-gl-layers is only meaningful on linux unittests, where it defaults to True 620 if task['test-platform'].startswith('linux') and task['suite'] not in ['talos', 'raptor']: 621 task.setdefault('allow-software-gl-layers', True) 622 else: 623 task['allow-software-gl-layers'] = False 624 625 # Enable WebRender by default on the QuantumRender test platforms, since 626 # the whole point of QuantumRender is to run with WebRender enabled. 627 # This currently matches linux64-qr and windows10-64-qr; both of these 628 # have /opt and /debug variants. 629 if "-qr/" in task['test-platform']: 630 task['webrender'] = True 631 else: 632 task.setdefault('webrender', False) 633 634 task.setdefault('e10s', True) 635 task.setdefault('try-name', task['test-name']) 636 task.setdefault('os-groups', []) 637 task.setdefault('run-as-administrator', False) 638 task.setdefault('chunks', 1) 639 task.setdefault('run-on-projects', 'built-projects') 640 task.setdefault('instance-size', 'default') 641 task.setdefault('max-run-time', 3600) 642 task.setdefault('reboot', False) 643 task.setdefault('virtualization', 'virtual') 644 task.setdefault('loopback-audio', False) 645 task.setdefault('loopback-video', False) 646 task.setdefault('limit-platforms', []) 647 # Bug 1602863 - temporarily in place while ubuntu1604 and ubuntu1804 648 # both exist in the CI. 649 if ('linux1804' in task['test-platform']): 650 task.setdefault('docker-image', {'in-tree': 'ubuntu1804-test'}) 651 else: 652 task.setdefault('docker-image', {'in-tree': 'desktop1604-test'}) 653 task.setdefault('checkout', False) 654 task.setdefault('require-signed-extensions', False) 655 task.setdefault('variants', []) 656 657 task['mozharness'].setdefault('extra-options', []) 658 task['mozharness'].setdefault('requires-signed-builds', False) 659 task['mozharness'].setdefault('tooltool-downloads', 'public') 660 task['mozharness'].setdefault('set-moz-node-path', False) 661 task['mozharness'].setdefault('chunked', False) 662 yield task 663 664 665@transforms.add 666def resolve_keys(config, tasks): 667 for task in tasks: 668 resolve_keyed_by( 669 task, 'require-signed-extensions', 670 item_name=task['test-name'], 671 **{ 672 'release-type': config.params['release_type'], 673 } 674 ) 675 yield task 676 677 678@transforms.add 679def setup_raptor(config, tasks): 680 """Add options that are specific to raptor jobs (identified by suite=raptor)""" 681 from taskgraph.transforms.raptor import transforms as raptor_transforms 682 683 for task in tasks: 684 if task['suite'] != 'raptor': 685 yield task 686 continue 687 688 for t in raptor_transforms(config, [task]): 689 yield t 690 691 692@transforms.add 693def limit_platforms(config, tasks): 694 for task in tasks: 695 if not task['limit-platforms']: 696 yield task 697 continue 698 699 limited_platforms = {key: key for key in task['limit-platforms']} 700 if keymatch(limited_platforms, task['test-platform']): 701 yield task 702 703 704transforms.add_validate(test_description_schema) 705 706 707@transforms.add 708def handle_suite_category(config, tasks): 709 for task in tasks: 710 task.setdefault('suite', {}) 711 712 if isinstance(task['suite'], text_type): 713 task['suite'] = {'name': task['suite']} 714 715 suite = task['suite'].setdefault('name', task['test-name']) 716 category = task['suite'].setdefault('category', suite) 717 718 task.setdefault('attributes', {}) 719 task['attributes']['unittest_suite'] = suite 720 task['attributes']['unittest_category'] = category 721 722 script = task['mozharness']['script'] 723 category_arg = None 724 if suite.startswith('test-verify') or suite.startswith('test-coverage'): 725 pass 726 elif script in ('android_emulator_unittest.py', 'android_hardware_unittest.py'): 727 category_arg = '--test-suite' 728 elif script == 'desktop_unittest.py': 729 category_arg = '--{}-suite'.format(category) 730 731 if category_arg: 732 task['mozharness'].setdefault('extra-options', []) 733 extra = task['mozharness']['extra-options'] 734 if not any(arg.startswith(category_arg) for arg in extra): 735 extra.append('{}={}'.format(category_arg, suite)) 736 737 # From here on out we only use the suite name. 738 task['suite'] = suite 739 yield task 740 741 742@transforms.add 743def setup_talos(config, tasks): 744 """Add options that are specific to talos jobs (identified by suite=talos)""" 745 for task in tasks: 746 if task['suite'] != 'talos': 747 yield task 748 continue 749 750 extra_options = task.setdefault('mozharness', {}).setdefault('extra-options', []) 751 extra_options.append('--use-talos-json') 752 753 # win7 needs to test skip 754 if task['build-platform'].startswith('win32'): 755 extra_options.append('--add-option') 756 extra_options.append('--setpref,gfx.direct2d.disabled=true') 757 758 yield task 759 760 761@transforms.add 762def setup_browsertime_flag(config, tasks): 763 """Optionally add `--browsertime` flag to Raptor pageload tests.""" 764 765 browsertime_flag = config.params['try_task_config'].get('browsertime', False) 766 767 for task in tasks: 768 if not browsertime_flag or task['suite'] != 'raptor': 769 yield task 770 continue 771 772 if task['treeherder-symbol'].startswith('Rap'): 773 # The Rap group is subdivided as Rap{-fenix,-refbrow,-fennec}(...), 774 # so `taskgraph.util.treeherder.replace_group` isn't appropriate. 775 task['treeherder-symbol'] = task['treeherder-symbol'].replace('Rap', 'Btime', 1) 776 777 extra_options = task.setdefault('mozharness', {}).setdefault('extra-options', []) 778 extra_options.append('--browsertime') 779 780 yield task 781 782 783@transforms.add 784def handle_artifact_prefix(config, tasks): 785 """Handle translating `artifact_prefix` appropriately""" 786 for task in tasks: 787 if task['build-attributes'].get('artifact_prefix'): 788 task.setdefault("attributes", {}).setdefault( 789 'artifact_prefix', task['build-attributes']['artifact_prefix'] 790 ) 791 yield task 792 793 794@transforms.add 795def set_target(config, tasks): 796 for task in tasks: 797 build_platform = task['build-platform'] 798 target = None 799 if 'target' in task: 800 resolve_keyed_by(task, 'target', item_name=task['test-name']) 801 target = task['target'] 802 if not target: 803 if build_platform.startswith('macosx'): 804 target = 'target.dmg' 805 elif build_platform.startswith('android'): 806 target = 'target.apk' 807 elif build_platform.startswith('win'): 808 target = 'target.zip' 809 else: 810 target = 'target.tar.bz2' 811 812 if isinstance(target, dict): 813 # TODO Remove hardcoded mobile artifact prefix 814 index_url = get_index_url(target['index']) 815 installer_url = '{}/artifacts/public/{}'.format(index_url, target['name']) 816 task['mozharness']['installer-url'] = installer_url 817 else: 818 task['mozharness']['build-artifact-name'] = get_artifact_path(task, target) 819 820 yield task 821 822 823@transforms.add 824def set_treeherder_machine_platform(config, tasks): 825 """Set the appropriate task.extra.treeherder.machine.platform""" 826 translation = { 827 # Linux64 build platforms for asan and pgo are specified differently to 828 # treeherder. 829 'linux64-pgo/opt': 'linux64/pgo', 830 'macosx1014-64/debug': 'osx-10-14/debug', 831 'macosx1014-64/opt': 'osx-10-14/opt', 832 'macosx1014-64-shippable/opt': 'osx-10-14-shippable/opt', 833 'win64-asan/opt': 'windows10-64/asan', 834 'win64-aarch64/opt': 'windows10-aarch64/opt', 835 'win32-pgo/opt': 'windows7-32/pgo', 836 'win64-pgo/opt': 'windows10-64/pgo', 837 } 838 for task in tasks: 839 # For most desktop platforms, the above table is not used for "regular" 840 # builds, so we'll always pick the test platform here. 841 # On macOS though, the regular builds are in the table. This causes a 842 # conflict in `verify_task_graph_symbol` once you add a new test 843 # platform based on regular macOS builds, such as for QR. 844 # Since it's unclear if the regular macOS builds can be removed from 845 # the table, workaround the issue for QR. 846 if 'android' in task['test-platform'] and 'pgo/opt' in task['test-platform']: 847 platform_new = task['test-platform'].replace('-pgo/opt', '/pgo') 848 task['treeherder-machine-platform'] = platform_new 849 elif 'android-em-7.0-x86_64-qr' in task['test-platform']: 850 opt = task['test-platform'].split('/')[1] 851 task['treeherder-machine-platform'] = 'android-em-7-0-x86_64-qr/'+opt 852 elif '-qr' in task['test-platform']: 853 task['treeherder-machine-platform'] = task['test-platform'] 854 elif 'android-hw' in task['test-platform']: 855 task['treeherder-machine-platform'] = task['test-platform'] 856 elif 'android-em-7.0-x86_64' in task['test-platform']: 857 opt = task['test-platform'].split('/')[1] 858 task['treeherder-machine-platform'] = 'android-em-7-0-x86_64/'+opt 859 elif 'android-em-7.0-x86' in task['test-platform']: 860 opt = task['test-platform'].split('/')[1] 861 task['treeherder-machine-platform'] = 'android-em-7-0-x86/'+opt 862 # Bug 1602863 - must separately define linux64/asan and linux1804-64/asan 863 # otherwise causes an exception during taskgraph generation about 864 # duplicate treeherder platform/symbol. 865 elif 'linux64-asan/opt' in task['test-platform']: 866 task['treeherder-machine-platform'] = 'linux64/asan' 867 elif 'linux1804-asan/opt' in task['test-platform']: 868 task['treeherder-machine-platform'] = 'linux1804-64/asan' 869 else: 870 task['treeherder-machine-platform'] = translation.get( 871 task['build-platform'], task['test-platform']) 872 yield task 873 874 875@transforms.add 876def set_tier(config, tasks): 877 """Set the tier based on policy for all test descriptions that do not 878 specify a tier otherwise.""" 879 for task in tasks: 880 if 'tier' in task: 881 resolve_keyed_by(task, 'tier', item_name=task['test-name']) 882 883 if 'fission-tier' in task: 884 resolve_keyed_by(task, 'fission-tier', item_name=task['test-name']) 885 886 # only override if not set for the test 887 if 'tier' not in task or task['tier'] == 'default': 888 if task['test-platform'] in [ 889 'linux64/opt', 890 'linux64/debug', 891 'linux64-pgo/opt', 892 'linux64-shippable/opt', 893 'linux64-devedition/opt', 894 'linux64-asan/opt', 895 'linux64-qr/opt', 896 'linux64-qr/debug', 897 'linux64-pgo-qr/opt', 898 'linux64-shippable-qr/opt', 899 'linux1804-64/opt', 900 'linux1804-64/debug', 901 'linux1804-64-shippable/opt', 902 'linux1804-64-qr/opt', 903 'linux1804-64-qr/debug', 904 'linux1804-64-shippable-qr/opt', 905 'linux1804-64-asan/opt', 906 'linux1804-64-tsan/opt', 907 'windows7-32/debug', 908 'windows7-32/opt', 909 'windows7-32-pgo/opt', 910 'windows7-32-devedition/opt', 911 'windows7-32-shippable/opt', 912 'windows10-aarch64/opt', 913 'windows10-64/debug', 914 'windows10-64/opt', 915 'windows10-64-pgo/opt', 916 'windows10-64-shippable/opt', 917 'windows10-64-devedition/opt', 918 'windows10-64-asan/opt', 919 'windows10-64-qr/opt', 920 'windows10-64-qr/debug', 921 'windows10-64-pgo-qr/opt', 922 'windows10-64-shippable-qr/opt', 923 'macosx1014-64/opt', 924 'macosx1014-64/debug', 925 'macosx1014-64-shippable/opt', 926 'macosx1014-64-devedition/opt', 927 'macosx1014-64-qr/opt', 928 'macosx1014-64-shippable-qr/opt', 929 'macosx1014-64-qr/debug', 930 'android-em-7.0-x86_64/opt', 931 'android-em-7.0-x86_64/debug', 932 'android-em-7.0-x86/opt', 933 'android-em-7.0-x86_64-qr/opt', 934 'android-em-7.0-x86_64-qr/debug' 935 ]: 936 task['tier'] = 1 937 else: 938 task['tier'] = 2 939 940 yield task 941 942 943@transforms.add 944def set_expires_after(config, tasks): 945 """Try jobs expire after 2 weeks; everything else lasts 1 year. This helps 946 keep storage costs low.""" 947 for task in tasks: 948 if 'expires-after' not in task: 949 if config.params.is_try(): 950 task['expires-after'] = "14 days" 951 else: 952 task['expires-after'] = "1 year" 953 yield task 954 955 956@transforms.add 957def set_download_symbols(config, tasks): 958 """In general, we download symbols immediately for debug builds, but only 959 on demand for everything else. ASAN builds shouldn't download 960 symbols since they don't product symbol zips see bug 1283879""" 961 for task in tasks: 962 if task['test-platform'].split('/')[-1] == 'debug': 963 task['mozharness']['download-symbols'] = True 964 elif task['build-platform'] == 'linux64-asan/opt' or \ 965 task['build-platform'] == 'windows10-64-asan/opt': 966 if 'download-symbols' in task['mozharness']: 967 del task['mozharness']['download-symbols'] 968 else: 969 task['mozharness']['download-symbols'] = 'ondemand' 970 yield task 971 972 973@transforms.add 974def handle_keyed_by(config, tasks): 975 """Resolve fields that can be keyed by platform, etc.""" 976 fields = [ 977 'instance-size', 978 'docker-image', 979 'max-run-time', 980 'chunks', 981 'variants', 982 'e10s', 983 'suite', 984 'run-on-projects', 985 'fission-run-on-projects', 986 'os-groups', 987 'run-as-administrator', 988 'workdir', 989 'worker-type', 990 'virtualization', 991 'fetches.fetch', 992 'fetches.toolchain', 993 'target', 994 ] 995 for task in tasks: 996 for field in fields: 997 resolve_keyed_by(task, field, item_name=task['test-name'], 998 project=config.params['project']) 999 yield task 1000 1001 1002@transforms.add 1003def setup_browsertime(config, tasks): 1004 """Configure browsertime dependencies for Raptor pageload tests that have 1005`--browsertime` extra option.""" 1006 1007 for task in tasks: 1008 # We need to make non-trivial changes to various fetches, and our 1009 # `by-test-platform` may not be "compatible" with existing 1010 # `by-test-platform` filters. Therefore we do everything after 1011 # `handle_keyed_by` so that existing fields have been resolved down to 1012 # simple lists. But we use the `by-test-platform` machinery to express 1013 # filters so that when the time comes to move browsertime into YAML 1014 # files, the transition is straight-forward. 1015 extra_options = task.get('mozharness', {}).get('extra-options', []) 1016 1017 if task['suite'] != 'raptor' or '--browsertime' not in extra_options: 1018 yield task 1019 continue 1020 1021 # This is appropriate as the browsertime task variants mature. 1022 task['tier'] = max(task['tier'], 1) 1023 1024 ts = { 1025 'by-test-platform': { 1026 'android.*': ['browsertime', 'linux64-geckodriver', 'linux64-node'], 1027 'linux.*': ['browsertime', 'linux64-geckodriver', 'linux64-node'], 1028 'macosx.*': ['browsertime', 'macosx64-geckodriver', 'macosx64-node'], 1029 'windows.*aarch64.*': ['browsertime', 'win32-geckodriver', 'win32-node'], 1030 'windows.*-32.*': ['browsertime', 'win32-geckodriver', 'win32-node'], 1031 'windows.*-64.*': ['browsertime', 'win64-geckodriver', 'win64-node'], 1032 }, 1033 } 1034 1035 task.setdefault('fetches', {}).setdefault('toolchain', []).extend( 1036 evaluate_keyed_by(ts, 'fetches.toolchain', task)) 1037 1038 fs = { 1039 'by-test-platform': { 1040 'android.*': ['linux64-ffmpeg-4.1.4'], 1041 'linux.*': ['linux64-ffmpeg-4.1.4'], 1042 'macosx.*': ['mac64-ffmpeg-4.1.1'], 1043 'windows.*aarch64.*': ['win64-ffmpeg-4.1.1'], 1044 'windows.*-32.*': ['win64-ffmpeg-4.1.1'], 1045 'windows.*-64.*': ['win64-ffmpeg-4.1.1'], 1046 }, 1047 } 1048 1049 cd_fetches = { 1050 'android.*': [ 1051 'linux64-chromedriver-80', 1052 'linux64-chromedriver-81' 1053 ], 1054 'linux.*': [ 1055 'linux64-chromedriver-79', 1056 'linux64-chromedriver-80', 1057 'linux64-chromedriver-81' 1058 ], 1059 'macosx.*': [ 1060 'mac64-chromedriver-79', 1061 'mac64-chromedriver-80', 1062 'mac64-chromedriver-81' 1063 ], 1064 'windows.*aarch64.*': [ 1065 'win32-chromedriver-79', 1066 'win32-chromedriver-80', 1067 'win32-chromedriver-81' 1068 ], 1069 'windows.*-32.*': [ 1070 'win32-chromedriver-79', 1071 'win32-chromedriver-80', 1072 'win32-chromedriver-81' 1073 ], 1074 'windows.*-64.*': [ 1075 'win32-chromedriver-79', 1076 'win32-chromedriver-80', 1077 'win32-chromedriver-81' 1078 ], 1079 } 1080 1081 if '--app=chrome' in extra_options \ 1082 or '--app=chromium' in extra_options \ 1083 or '--app=chrome-m' in extra_options: 1084 # Only add the chromedriver fetches when chrome/chromium is running 1085 for platform in cd_fetches: 1086 fs['by-test-platform'][platform].extend(cd_fetches[platform]) 1087 1088 # Disable the Raptor install step 1089 if '--app=chrome-m' in extra_options: 1090 extra_options.append('--noinstall') 1091 1092 task.setdefault('fetches', {}).setdefault('fetch', []).extend( 1093 evaluate_keyed_by(fs, 'fetches.fetch', task)) 1094 1095 extra_options.extend(('--browsertime-browsertimejs', 1096 '$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js')) # noqa: E501 1097 1098 eos = { 1099 'by-test-platform': { 1100 'windows.*': 1101 ['--browsertime-node', 1102 '$MOZ_FETCHES_DIR/node/node.exe', 1103 '--browsertime-geckodriver', 1104 '$MOZ_FETCHES_DIR/geckodriver.exe', 1105 '--browsertime-chromedriver', 1106 '$MOZ_FETCHES_DIR/{}chromedriver.exe', 1107 '--browsertime-ffmpeg', 1108 '$MOZ_FETCHES_DIR/ffmpeg-4.1.1-win64-static/bin/ffmpeg.exe', 1109 ], 1110 'macosx.*': 1111 ['--browsertime-node', 1112 '$MOZ_FETCHES_DIR/node/bin/node', 1113 '--browsertime-geckodriver', 1114 '$MOZ_FETCHES_DIR/geckodriver', 1115 '--browsertime-chromedriver', 1116 '$MOZ_FETCHES_DIR/{}chromedriver', 1117 '--browsertime-ffmpeg', 1118 '$MOZ_FETCHES_DIR/ffmpeg-4.1.1-macos64-static/bin/ffmpeg', 1119 ], 1120 'default': 1121 ['--browsertime-node', 1122 '$MOZ_FETCHES_DIR/node/bin/node', 1123 '--browsertime-geckodriver', 1124 '$MOZ_FETCHES_DIR/geckodriver', 1125 '--browsertime-chromedriver', 1126 '$MOZ_FETCHES_DIR/{}chromedriver', 1127 '--browsertime-ffmpeg', 1128 '$MOZ_FETCHES_DIR/ffmpeg-4.1.4-i686-static/ffmpeg', 1129 ], 1130 } 1131 } 1132 1133 extra_options.extend(evaluate_keyed_by(eos, 'mozharness.extra-options', task)) 1134 1135 yield task 1136 1137 1138def get_mobile_project(task): 1139 """Returns the mobile project of the specified task or None.""" 1140 1141 if not task['build-platform'].startswith('android'): 1142 return 1143 1144 mobile_projects = ( 1145 'fenix', 1146 'fennec', 1147 'geckoview', 1148 'refbrow', 1149 'chrome-m' 1150 ) 1151 1152 for name in mobile_projects: 1153 if name in task['test-name']: 1154 return name 1155 1156 target = task.get('target') 1157 if target: 1158 if isinstance(target, dict): 1159 target = target['name'] 1160 1161 for name in mobile_projects: 1162 if name in target: 1163 return name 1164 1165 return 'fennec' 1166 1167 1168@transforms.add 1169def disable_fennec_e10s(config, tasks): 1170 for task in tasks: 1171 if get_mobile_project(task) == 'fennec': 1172 # Fennec is non-e10s 1173 task['e10s'] = False 1174 yield task 1175 1176 1177@transforms.add 1178def disable_wpt_timeouts_on_autoland(config, tasks): 1179 """do not run web-platform-tests that are expected TIMEOUT on autoland""" 1180 for task in tasks: 1181 if 'web-platform-tests' in task['test-name'] and config.params['project'] == 'autoland': 1182 task['mozharness'].setdefault('extra-options', []).append('--skip-timeout') 1183 yield task 1184 1185 1186@transforms.add 1187def enable_code_coverage(config, tasks): 1188 """Enable code coverage for the ccov build-platforms""" 1189 for task in tasks: 1190 if 'ccov' in task['build-platform']: 1191 # Do not run tests on fuzzing builds 1192 if 'fuzzing' in task['build-platform']: 1193 task['run-on-projects'] = [] 1194 continue 1195 1196 # Skip this transform for android code coverage builds. 1197 if 'android' in task['build-platform']: 1198 task.setdefault('fetches', {}).setdefault('toolchain', []).append('linux64-grcov') 1199 task['mozharness'].setdefault('extra-options', []).append('--java-code-coverage') 1200 yield task 1201 continue 1202 task['mozharness'].setdefault('extra-options', []).append('--code-coverage') 1203 task['instance-size'] = 'xlarge' 1204 1205 # Temporarily disable Mac tests on mozilla-central 1206 if 'mac' in task['build-platform']: 1207 task['run-on-projects'] = ['try'] 1208 1209 # Ensure we always run on the projects defined by the build, unless the test 1210 # is try only or shouldn't run at all. 1211 if task['run-on-projects'] not in [[], ['try']]: 1212 task['run-on-projects'] = 'built-projects' 1213 1214 # Ensure we don't optimize test suites out. 1215 # We always want to run all test suites for coverage purposes. 1216 task.pop('schedules-component', None) 1217 task.pop('when', None) 1218 task['optimization'] = None 1219 1220 # Add a toolchain and a fetch task for the grcov binary. 1221 if any(p in task['build-platform'] for p in ('linux', 'osx', 'win')): 1222 task.setdefault('fetches', {}) 1223 task['fetches'].setdefault('fetch', []) 1224 task['fetches'].setdefault('toolchain', []) 1225 1226 if 'linux' in task['build-platform']: 1227 task['fetches']['toolchain'].append('linux64-grcov') 1228 elif 'osx' in task['build-platform']: 1229 task['fetches']['fetch'].append('grcov-osx-x86_64') 1230 elif 'win' in task['build-platform']: 1231 task['fetches']['toolchain'].append('win64-grcov') 1232 1233 if 'talos' in task['test-name']: 1234 task['max-run-time'] = 7200 1235 if 'linux' in task['build-platform']: 1236 task['docker-image'] = {"in-tree": "ubuntu1804-test"} 1237 task['mozharness']['extra-options'].append('--add-option') 1238 task['mozharness']['extra-options'].append('--cycles,1') 1239 task['mozharness']['extra-options'].append('--add-option') 1240 task['mozharness']['extra-options'].append('--tppagecycles,1') 1241 task['mozharness']['extra-options'].append('--add-option') 1242 task['mozharness']['extra-options'].append('--no-upload-results') 1243 task['mozharness']['extra-options'].append('--add-option') 1244 task['mozharness']['extra-options'].append('--tptimeout,15000') 1245 if 'raptor' in task['test-name']: 1246 task['max-run-time'] = 1800 1247 if 'linux' in task['build-platform']: 1248 task['docker-image'] = {"in-tree": "desktop1604-test"} 1249 yield task 1250 1251 1252@transforms.add 1253def handle_run_on_projects(config, tasks): 1254 """Handle translating `built-projects` appropriately""" 1255 for task in tasks: 1256 if task['run-on-projects'] == 'built-projects': 1257 task['run-on-projects'] = task['build-attributes'].get('run_on_projects', ['all']) 1258 yield task 1259 1260 1261@transforms.add 1262def split_variants(config, tasks): 1263 for task in tasks: 1264 variants = task.pop('variants', []) 1265 1266 yield copy.deepcopy(task) 1267 1268 for name in variants: 1269 taskv = copy.deepcopy(task) 1270 variant = TEST_VARIANTS[name] 1271 1272 if 'filterfn' in variant and not variant['filterfn'](taskv): 1273 continue 1274 1275 taskv['attributes']['unittest_variant'] = name 1276 taskv['description'] = variant['description'].format(**taskv) 1277 1278 suffix = '-' + variant['suffix'] 1279 taskv['test-name'] += suffix 1280 taskv['try-name'] += suffix 1281 1282 group, symbol = split_symbol(taskv['treeherder-symbol']) 1283 if group != '?': 1284 group += suffix 1285 else: 1286 symbol += suffix 1287 taskv['treeherder-symbol'] = join_symbol(group, symbol) 1288 1289 taskv.update(variant.get('replace', {})) 1290 1291 if task['suite'] == 'raptor': 1292 taskv['tier'] = max(taskv['tier'], 2) 1293 1294 yield merge(taskv, variant.get('merge', {})) 1295 1296 1297@transforms.add 1298def handle_fission_attributes(config, tasks): 1299 """Handle run_on_projects for fission tasks.""" 1300 for task in tasks: 1301 for attr in ('run-on-projects', 'tier'): 1302 fission_attr = task.pop('fission-{}'.format(attr), None) 1303 1304 if task['attributes'].get('unittest_variant') != 'fission' or fission_attr is None: 1305 continue 1306 1307 task[attr] = fission_attr 1308 1309 yield task 1310 1311 1312@transforms.add 1313def ensure_spi_disabled_on_all_but_spi(config, tasks): 1314 for task in tasks: 1315 variant = task['attributes'].get('unittest_variant', '') 1316 has_setpref = ('gtest' not in task['suite'] and 1317 'cppunit' not in task['suite'] and 1318 'jittest' not in task['suite'] and 1319 'junit' not in task['suite'] and 1320 'raptor' not in task['suite']) 1321 1322 if has_setpref and variant != 'socketprocess' and variant != 'socketprocess_networking': 1323 task['mozharness']['extra-options'].append( 1324 '--setpref=media.peerconnection.mtransport_process=false') 1325 task['mozharness']['extra-options'].append( 1326 '--setpref=network.process.enabled=false') 1327 1328 yield task 1329 1330 1331@transforms.add 1332def split_e10s(config, tasks): 1333 for task in tasks: 1334 e10s = task['e10s'] 1335 1336 if e10s: 1337 task_copy = copy.deepcopy(task) 1338 task_copy['test-name'] += '-e10s' 1339 task_copy['e10s'] = True 1340 task_copy['attributes']['e10s'] = True 1341 yield task_copy 1342 1343 if not e10s or e10s == 'both': 1344 task['test-name'] += '-1proc' 1345 task['try-name'] += '-1proc' 1346 task['e10s'] = False 1347 task['attributes']['e10s'] = False 1348 group, symbol = split_symbol(task['treeherder-symbol']) 1349 if group != '?': 1350 group += '-1proc' 1351 task['treeherder-symbol'] = join_symbol(group, symbol) 1352 task['mozharness']['extra-options'].append('--disable-e10s') 1353 yield task 1354 1355 1356@transforms.add 1357def set_test_verify_chunks(config, tasks): 1358 """Set the number of chunks we use for test-verify.""" 1359 for task in tasks: 1360 if any(task['suite'].startswith(s) for s in ('test-verify', 'test-coverage')): 1361 env = config.params.get('try_task_config', {}) or {} 1362 env = env.get('templates', {}).get('env', {}) 1363 task['chunks'] = perfile_number_of_chunks(config.params.is_try(), 1364 env.get('MOZHARNESS_TEST_PATHS', ''), 1365 config.params.get('head_repository', ''), 1366 config.params.get('head_rev', ''), 1367 task['test-name']) 1368 1369 # limit the number of chunks we run for test-verify mode because 1370 # test-verify is comprehensive and takes a lot of time, if we have 1371 # >30 tests changed, this is probably an import of external tests, 1372 # or a patch renaming/moving files in bulk 1373 maximum_number_verify_chunks = 3 1374 if task['chunks'] > maximum_number_verify_chunks: 1375 task['chunks'] = maximum_number_verify_chunks 1376 1377 yield task 1378 1379 1380@transforms.add 1381def set_test_manifests(config, tasks): 1382 """Determine the set of test manifests that should run in this task.""" 1383 1384 for task in tasks: 1385 if task['suite'] in CHUNK_SUITES_BLACKLIST: 1386 yield task 1387 continue 1388 1389 if taskgraph.fast: 1390 # We want to avoid evaluating manifests when taskgraph.fast is set. But 1391 # manifests are required for dynamic chunking. Just set the number of 1392 # chunks to one in this case. 1393 if task['chunks'] == 'dynamic': 1394 task['chunks'] = 1 1395 yield task 1396 continue 1397 1398 manifests = task.get('test-manifests') 1399 if manifests: 1400 if isinstance(manifests, list): 1401 task['test-manifests'] = {'active': manifests, 'skipped': []} 1402 yield task 1403 continue 1404 1405 suite_definition = TEST_SUITES[task['suite']] 1406 mozinfo = guess_mozinfo_from_task(task) 1407 1408 loader = manifest_loaders[config.params['test_manifest_loader']] 1409 task['test-manifests'] = loader.get_manifests( 1410 suite_definition['build_flavor'], 1411 suite_definition.get('kwargs', {}).get('subsuite', 'undefined'), 1412 frozenset(mozinfo.items()), 1413 ) 1414 1415 # Skip the task if the loader doesn't return any manifests for the 1416 # associated suite. 1417 if not task['test-manifests']['active'] and not task['test-manifests']['skipped']: 1418 continue 1419 1420 yield task 1421 1422 1423@transforms.add 1424def resolve_dynamic_chunks(config, tasks): 1425 """Determine how many chunks are needed to handle the given set of manifests.""" 1426 1427 for task in tasks: 1428 if task['chunks'] != "dynamic": 1429 yield task 1430 continue 1431 1432 if not task.get('test-manifests'): 1433 raise Exception( 1434 "{} must define 'test-manifests' to use dynamic chunking!".format( 1435 task['test-name'])) 1436 1437 runtimes = {m: r for m, r in get_runtimes(task['test-platform']).items() 1438 if m in task['test-manifests']['active']} 1439 1440 times = list(runtimes.values()) 1441 avg = round(sum(times) / len(times), 2) if times else 0 1442 total = sum(times) 1443 1444 # If there are manifests missing from the runtimes data, fill them in 1445 # with the average of all present manifests. 1446 missing = [m for m in task['test-manifests']['active'] if m not in runtimes] 1447 total += avg * len(missing) 1448 1449 task['chunks'] = int(round(total / DYNAMIC_CHUNK_DURATION)) or 1 1450 yield task 1451 1452 1453@transforms.add 1454def split_chunks(config, tasks): 1455 """Based on the 'chunks' key, split tests up into chunks by duplicating 1456 them and assigning 'this-chunk' appropriately and updating the treeherder 1457 symbol. 1458 """ 1459 1460 for task in tasks: 1461 # If test-manifests are set, chunk them ahead of time to avoid running 1462 # the algorithm more than once. 1463 chunked_manifests = None 1464 if 'test-manifests' in task: 1465 suite_definition = TEST_SUITES[task['suite']] 1466 manifests = task['test-manifests'] 1467 chunked_manifests = chunk_manifests( 1468 suite_definition['build_flavor'], 1469 suite_definition.get('kwargs', {}).get('subsuite', 'undefined'), 1470 task['test-platform'], 1471 task['chunks'], 1472 manifests['active'], 1473 ) 1474 1475 # Add all skipped manifests to the first chunk so they still show up in the 1476 # logs. They won't impact runtime much. 1477 chunked_manifests[0].extend(manifests['skipped']) 1478 1479 for i in range(task['chunks']): 1480 this_chunk = i + 1 1481 1482 # copy the test and update with the chunk number 1483 chunked = copy.deepcopy(task) 1484 chunked['this-chunk'] = this_chunk 1485 1486 if chunked_manifests is not None: 1487 manifests = sorted(chunked_manifests[i]) 1488 if not manifests: 1489 raise Exception( 1490 'Chunking algorithm yielded no manifests for chunk {} of {} on {}'.format( 1491 this_chunk, task['test-name'], task['test-platform'])) 1492 chunked['test-manifests'] = manifests 1493 1494 if task['chunks'] > 1: 1495 # add the chunk number to the TH symbol 1496 chunked['treeherder-symbol'] = add_suffix( 1497 chunked['treeherder-symbol'], this_chunk) 1498 1499 yield chunked 1500 1501 1502@transforms.add 1503def allow_software_gl_layers(config, tasks): 1504 """ 1505 Handle the "allow-software-gl-layers" property for platforms where it 1506 applies. 1507 """ 1508 for task in tasks: 1509 if task.get('allow-software-gl-layers'): 1510 # This should be set always once bug 1296086 is resolved. 1511 task['mozharness'].setdefault('extra-options', [])\ 1512 .append("--allow-software-gl-layers") 1513 1514 yield task 1515 1516 1517@transforms.add 1518def enable_webrender(config, tasks): 1519 """ 1520 Handle the "webrender" property by passing a flag to mozharness if it is 1521 enabled. 1522 """ 1523 for task in tasks: 1524 if task.get('webrender'): 1525 extra_options = task['mozharness'].setdefault('extra-options', []) 1526 extra_options.append("--enable-webrender") 1527 # We only want to 'setpref' on tests that have a profile 1528 if not task['attributes']['unittest_category'] in ['cppunittest', 'gtest', 'raptor']: 1529 extra_options.append("--setpref=layers.d3d11.enable-blacklist=false") 1530 1531 yield task 1532 1533 1534@transforms.add 1535def set_schedules_for_webrender_android(config, tasks): 1536 """android-hw has limited resources, we need webrender on phones""" 1537 for task in tasks: 1538 if task['suite'] in ['crashtest', 'reftest'] and \ 1539 task['test-platform'].startswith('android-hw'): 1540 task['schedules-component'] = 'android-hw-gfx' 1541 yield task 1542 1543 1544@transforms.add 1545def set_retry_exit_status(config, tasks): 1546 """Set the retry exit status to TBPL_RETRY, the value returned by mozharness 1547 scripts to indicate a transient failure that should be retried.""" 1548 for task in tasks: 1549 task['retry-exit-status'] = [4] 1550 yield task 1551 1552 1553@transforms.add 1554def set_profile(config, tasks): 1555 """Set profiling mode for tests.""" 1556 profile = config.params['try_task_config'].get('gecko-profile', False) 1557 1558 for task in tasks: 1559 if profile and task['suite'] in ['talos', 'raptor']: 1560 task['mozharness']['extra-options'].append('--geckoProfile') 1561 yield task 1562 1563 1564@transforms.add 1565def set_tag(config, tasks): 1566 """Set test for a specific tag.""" 1567 tag = None 1568 if config.params['try_mode'] == 'try_option_syntax': 1569 tag = config.params['try_options']['tag'] 1570 for task in tasks: 1571 if tag: 1572 task['mozharness']['extra-options'].extend(['--tag', tag]) 1573 yield task 1574 1575 1576@transforms.add 1577def set_test_type(config, tasks): 1578 types = ['mochitest', 'reftest', 'talos', 'raptor', 'geckoview-junit', 'gtest'] 1579 for task in tasks: 1580 for test_type in types: 1581 if test_type in task['suite'] and 'web-platform' not in task['suite']: 1582 task.setdefault('tags', {})['test-type'] = test_type 1583 yield task 1584 1585 1586@transforms.add 1587def set_worker_type(config, tasks): 1588 """Set the worker type based on the test platform.""" 1589 for task in tasks: 1590 # during the taskcluster migration, this is a bit tortured, but it 1591 # will get simpler eventually! 1592 test_platform = task['test-platform'] 1593 if task.get('worker-type'): 1594 # This test already has its worker type defined, so just use that (yields below) 1595 pass 1596 elif test_platform.startswith('macosx1014-64'): 1597 if '--power-test' in task['mozharness']['extra-options']: 1598 task['worker-type'] = MACOSX_WORKER_TYPES['macosx1014-64-power'] 1599 else: 1600 task['worker-type'] = MACOSX_WORKER_TYPES['macosx1014-64'] 1601 elif test_platform.startswith('win'): 1602 # figure out what platform the job needs to run on 1603 if task['virtualization'] == 'hardware': 1604 # some jobs like talos and reftest run on real h/w - those are all win10 1605 if test_platform.startswith('windows10-64-ref-hw-2017'): 1606 win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-64-ref-hw-2017'] 1607 elif test_platform.startswith('windows10-aarch64'): 1608 win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-aarch64'] 1609 else: 1610 win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-64'] 1611 else: 1612 # the other jobs run on a vm which may or may not be a win10 vm 1613 win_worker_type_platform = WINDOWS_WORKER_TYPES[ 1614 test_platform.split('/')[0] 1615 ] 1616 # now we have the right platform set the worker type accordingly 1617 task['worker-type'] = win_worker_type_platform[task['virtualization']] 1618 elif test_platform.startswith('android-hw-g5'): 1619 if task['suite'] != 'raptor': 1620 task['worker-type'] = 't-bitbar-gw-unit-g5' 1621 else: 1622 task['worker-type'] = 't-bitbar-gw-perf-g5' 1623 elif test_platform.startswith('android-hw-p2'): 1624 if task['suite'] != 'raptor': 1625 task['worker-type'] = 't-bitbar-gw-unit-p2' 1626 else: 1627 task['worker-type'] = 't-bitbar-gw-perf-p2' 1628 elif test_platform.startswith('android-em-7.0-x86'): 1629 task['worker-type'] = 't-linux-metal' 1630 elif test_platform.startswith('linux') or test_platform.startswith('android'): 1631 if task.get('suite', '') in ['talos', 'raptor'] and \ 1632 not task['build-platform'].startswith('linux64-ccov'): 1633 task['worker-type'] = 't-linux-talos' 1634 else: 1635 task['worker-type'] = LINUX_WORKER_TYPES[task['instance-size']] 1636 else: 1637 raise Exception("unknown test_platform {}".format(test_platform)) 1638 1639 yield task 1640 1641 1642@transforms.add 1643def set_schedules_components(config, tasks): 1644 for task in tasks: 1645 if 'optimization' in task or 'when' in task: 1646 yield task 1647 continue 1648 1649 category = task['attributes']['unittest_category'] 1650 schedules = task.get('schedules-component', category) 1651 if isinstance(schedules, string_types): 1652 schedules = [schedules] 1653 1654 schedules = set(schedules) 1655 if schedules & set(INCLUSIVE_COMPONENTS): 1656 # if this is an "inclusive" test, then all files which might 1657 # cause it to run are annotated with SCHEDULES in moz.build, 1658 # so do not include the platform or any other components here 1659 task['schedules-component'] = sorted(schedules) 1660 yield task 1661 continue 1662 1663 schedules.add(category) 1664 schedules.add(platform_family(task['build-platform'])) 1665 1666 if task['webrender']: 1667 schedules.add('webrender') 1668 1669 task['schedules-component'] = sorted(schedules) 1670 yield task 1671 1672 1673@transforms.add 1674def make_job_description(config, tasks): 1675 """Convert *test* descriptions to *job* descriptions (input to 1676 taskgraph.transforms.job)""" 1677 1678 for task in tasks: 1679 mobile = get_mobile_project(task) 1680 if mobile and (mobile not in task['test-name']): 1681 label = '{}-{}-{}-{}'.format(config.kind, task['test-platform'], mobile, 1682 task['test-name']) 1683 else: 1684 label = '{}-{}-{}'.format(config.kind, task['test-platform'], task['test-name']) 1685 if task['chunks'] > 1: 1686 label += '-{}'.format(task['this-chunk']) 1687 1688 build_label = task['build-label'] 1689 1690 try_name = task['try-name'] 1691 if task['suite'] == 'talos': 1692 attr_try_name = 'talos_try_name' 1693 elif task['suite'] == 'raptor': 1694 attr_try_name = 'raptor_try_name' 1695 else: 1696 attr_try_name = 'unittest_try_name' 1697 1698 attr_build_platform, attr_build_type = task['build-platform'].split('/', 1) 1699 1700 attributes = task.get('attributes', {}) 1701 attributes.update({ 1702 'build_platform': attr_build_platform, 1703 'build_type': attr_build_type, 1704 'test_platform': task['test-platform'], 1705 'test_chunk': str(task['this-chunk']), 1706 'test_manifests': task.get('test-manifests'), 1707 attr_try_name: try_name, 1708 }) 1709 1710 jobdesc = {} 1711 name = '{}-{}'.format(task['test-platform'], task['test-name']) 1712 jobdesc['name'] = name 1713 jobdesc['label'] = label 1714 jobdesc['description'] = task['description'] 1715 jobdesc['attributes'] = attributes 1716 jobdesc['dependencies'] = {'build': build_label} 1717 jobdesc['job-from'] = task['job-from'] 1718 1719 if task.get('fetches'): 1720 jobdesc['fetches'] = task['fetches'] 1721 1722 if task['mozharness']['requires-signed-builds'] is True: 1723 jobdesc['dependencies']['build-signing'] = task['build-signing-label'] 1724 1725 jobdesc['expires-after'] = task['expires-after'] 1726 jobdesc['routes'] = [] 1727 jobdesc['run-on-projects'] = sorted(task['run-on-projects']) 1728 jobdesc['scopes'] = [] 1729 jobdesc['tags'] = task.get('tags', {}) 1730 jobdesc['extra'] = { 1731 'chunks': { 1732 'current': task['this-chunk'], 1733 'total': task['chunks'], 1734 }, 1735 'suite': attributes['unittest_suite'], 1736 } 1737 jobdesc['treeherder'] = { 1738 'symbol': task['treeherder-symbol'], 1739 'kind': 'test', 1740 'tier': task['tier'], 1741 'platform': task.get('treeherder-machine-platform', task['build-platform']), 1742 } 1743 1744 schedules = task.get('schedules-component', []) 1745 if task.get('when'): 1746 # This may still be used by comm-central. 1747 jobdesc['when'] = task['when'] 1748 elif 'optimization' in task: 1749 jobdesc['optimization'] = task['optimization'] 1750 # Pushes generated by `mach try auto` should use the non-try optimizations. 1751 elif config.params.is_try() and config.params['try_mode'] != 'try_auto': 1752 jobdesc['optimization'] = {'test-try': schedules} 1753 elif set(schedules) & set(INCLUSIVE_COMPONENTS): 1754 jobdesc['optimization'] = {'test-inclusive': schedules} 1755 else: 1756 # First arg goes to 'skip-unless-schedules', second goes to the 1757 # main test strategy. Using an empty dict allows earlier 1758 # substrategies (of a CompositeStrategy) to pass values by reference 1759 # to later substrategies. 1760 jobdesc['optimization'] = {'test': (schedules, {})} 1761 1762 run = jobdesc['run'] = {} 1763 run['using'] = 'mozharness-test' 1764 run['test'] = task 1765 1766 if 'workdir' in task: 1767 run['workdir'] = task.pop('workdir') 1768 1769 jobdesc['worker-type'] = task.pop('worker-type') 1770 if task.get('fetches'): 1771 jobdesc['fetches'] = task.pop('fetches') 1772 1773 yield jobdesc 1774 1775 1776def normpath(path): 1777 return path.replace('/', '\\') 1778 1779 1780def get_firefox_version(): 1781 with open('browser/config/version.txt', 'r') as f: 1782 return f.readline().strip() 1783