1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7""" 8run talos tests in a virtualenv 9""" 10 11from __future__ import absolute_import 12import six 13import io 14import os 15import sys 16import pprint 17import copy 18import re 19import shutil 20import subprocess 21import json 22 23import mozharness 24from mozharness.base.config import parse_config_file 25from mozharness.base.errors import PythonErrorList 26from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL 27from mozharness.base.log import INFO, WARNING 28from mozharness.base.python import Python3Virtualenv 29from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 30from mozharness.base.vcs.vcsbase import MercurialScript 31from mozharness.mozilla.testing.errors import TinderBoxPrintRe 32from mozharness.mozilla.automation import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE 33from mozharness.mozilla.automation import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING 34from mozharness.mozilla.tooltool import TooltoolMixin 35from mozharness.mozilla.testing.codecoverage import ( 36 CodeCoverageMixin, 37 code_coverage_config_options, 38) 39 40 41scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) 42external_tools_path = os.path.join(scripts_path, "external_tools") 43 44TalosErrorList = PythonErrorList + [ 45 {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG}, 46 {"substr": r"""FAIL: Graph server unreachable""", "level": CRITICAL}, 47 {"substr": r"""FAIL: Busted:""", "level": CRITICAL}, 48 {"substr": r"""FAIL: failed to cleanup""", "level": ERROR}, 49 {"substr": r"""erfConfigurator.py: Unknown error""", "level": CRITICAL}, 50 {"substr": r"""talosError""", "level": CRITICAL}, 51 { 52 "regex": re.compile(r"""No machine_name called '.*' can be found"""), 53 "level": CRITICAL, 54 }, 55 { 56 "substr": r"""No such file or directory: 'browser_output.txt'""", 57 "level": CRITICAL, 58 "explanation": "Most likely the browser failed to launch, or the test was otherwise " 59 "unsuccessful in even starting.", 60 }, 61] 62 63GeckoProfilerSettings = ( 64 "gecko_profile_interval", 65 "gecko_profile_entries", 66 "gecko_profile_features", 67 "gecko_profile_threads", 68) 69 70# TODO: check for running processes on script invocation 71 72 73class TalosOutputParser(OutputParser): 74 minidump_regex = re.compile( 75 r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''' 76 ) 77 RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})") 78 worst_tbpl_status = TBPL_SUCCESS 79 80 def __init__(self, **kwargs): 81 super(TalosOutputParser, self).__init__(**kwargs) 82 self.minidump_output = None 83 self.found_perf_data = [] 84 85 def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level): 86 self.worst_log_level = self.worst_level(log_level, self.worst_log_level) 87 self.worst_tbpl_status = self.worst_level( 88 tbpl_level, self.worst_tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE 89 ) 90 91 def parse_single_line(self, line): 92 """In Talos land, every line that starts with RETURN: needs to be 93 printed with a TinderboxPrint:""" 94 if line.startswith("RETURN:"): 95 line.replace("RETURN:", "TinderboxPrint:") 96 m = self.minidump_regex.search(line) 97 if m: 98 self.minidump_output = (m.group(1), m.group(2), m.group(3)) 99 100 m = self.RE_PERF_DATA.match(line) 101 if m: 102 self.found_perf_data.append(m.group(1)) 103 104 # now let's check if we should retry 105 harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] 106 if harness_retry_re.search(line): 107 self.critical(" %s" % line) 108 self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY) 109 return # skip base parse_single_line 110 super(TalosOutputParser, self).parse_single_line(line) 111 112 113class Talos( 114 TestingMixin, MercurialScript, TooltoolMixin, Python3Virtualenv, CodeCoverageMixin 115): 116 """ 117 install and run Talos tests 118 """ 119 120 config_options = ( 121 [ 122 [ 123 ["--use-talos-json"], 124 { 125 "action": "store_true", 126 "dest": "use_talos_json", 127 "default": False, 128 "help": "Use talos config from talos.json", 129 }, 130 ], 131 [ 132 ["--suite"], 133 { 134 "action": "store", 135 "dest": "suite", 136 "help": "Talos suite to run (from talos json)", 137 }, 138 ], 139 [ 140 ["--system-bits"], 141 { 142 "action": "store", 143 "dest": "system_bits", 144 "type": "choice", 145 "default": "32", 146 "choices": ["32", "64"], 147 "help": "Testing 32 or 64 (for talos json plugins)", 148 }, 149 ], 150 [ 151 ["--add-option"], 152 { 153 "action": "extend", 154 "dest": "talos_extra_options", 155 "default": None, 156 "help": "extra options to talos", 157 }, 158 ], 159 [ 160 ["--gecko-profile"], 161 { 162 "dest": "gecko_profile", 163 "action": "store_true", 164 "default": False, 165 "help": "Whether or not to profile the test run and save the profile results", 166 }, 167 ], 168 [ 169 ["--gecko-profile-interval"], 170 { 171 "dest": "gecko_profile_interval", 172 "type": "int", 173 "help": "The interval between samples taken by the profiler (milliseconds)", 174 }, 175 ], 176 [ 177 ["--gecko-profile-entries"], 178 { 179 "dest": "gecko_profile_entries", 180 "type": "int", 181 "help": "How many samples to take with the profiler", 182 }, 183 ], 184 [ 185 ["--gecko-profile-features"], 186 { 187 "dest": "gecko_profile_features", 188 "type": "str", 189 "default": None, 190 "help": "The features to enable in the profiler (comma-separated)", 191 }, 192 ], 193 [ 194 ["--gecko-profile-threads"], 195 { 196 "dest": "gecko_profile_threads", 197 "type": "str", 198 "help": "Comma-separated list of threads to sample.", 199 }, 200 ], 201 [ 202 ["--disable-e10s"], 203 { 204 "dest": "e10s", 205 "action": "store_false", 206 "default": True, 207 "help": "Run without multiple processes (e10s).", 208 }, 209 ], 210 [ 211 ["--disable-fission"], 212 { 213 "action": "store_false", 214 "dest": "fission", 215 "default": True, 216 "help": "Disable Fission (site isolation) in Gecko.", 217 }, 218 ], 219 [ 220 ["--setpref"], 221 { 222 "action": "append", 223 "metavar": "PREF=VALUE", 224 "dest": "extra_prefs", 225 "default": [], 226 "help": "Set a browser preference. May be used multiple times.", 227 }, 228 ], 229 [ 230 ["--skip-preflight"], 231 { 232 "action": "store_true", 233 "dest": "skip_preflight", 234 "default": False, 235 "help": "skip preflight commands to prepare machine.", 236 }, 237 ], 238 ] 239 + testing_config_options 240 + copy.deepcopy(code_coverage_config_options) 241 ) 242 243 def __init__(self, **kwargs): 244 kwargs.setdefault("config_options", self.config_options) 245 kwargs.setdefault( 246 "all_actions", 247 [ 248 "clobber", 249 "download-and-extract", 250 "populate-webroot", 251 "create-virtualenv", 252 "install", 253 "run-tests", 254 ], 255 ) 256 kwargs.setdefault( 257 "default_actions", 258 [ 259 "clobber", 260 "download-and-extract", 261 "populate-webroot", 262 "create-virtualenv", 263 "install", 264 "run-tests", 265 ], 266 ) 267 kwargs.setdefault("config", {}) 268 super(Talos, self).__init__(**kwargs) 269 270 self.workdir = self.query_abs_dirs()["abs_work_dir"] # convenience 271 272 self.run_local = self.config.get("run_local") 273 self.installer_url = self.config.get("installer_url") 274 self.test_packages_url = self.config.get("test_packages_url") 275 self.talos_json_url = self.config.get("talos_json_url") 276 self.talos_json = self.config.get("talos_json") 277 self.talos_json_config = self.config.get("talos_json_config") 278 self.repo_path = self.config.get("repo_path") 279 self.obj_path = self.config.get("obj_path") 280 self.tests = None 281 extra_opts = self.config.get("talos_extra_options", []) 282 self.gecko_profile = ( 283 self.config.get("gecko_profile") or "--gecko-profile" in extra_opts 284 ) 285 for setting in GeckoProfilerSettings: 286 value = self.config.get(setting) 287 arg = "--" + setting.replace("_", "-") 288 if value is None: 289 try: 290 value = extra_opts[extra_opts.index(arg) + 1] 291 except ValueError: 292 pass # Not found 293 if value is not None: 294 setattr(self, setting, value) 295 if not self.gecko_profile: 296 self.warning("enabling Gecko profiler for %s setting!" % setting) 297 self.gecko_profile = True 298 self.pagesets_name = None 299 self.benchmark_zip = None 300 self.webextensions_zip = None 301 302 # We accept some configuration options from the try commit message in the format 303 # mozharness: <options> 304 # Example try commit message: 305 # mozharness: --gecko-profile try: <stuff> 306 def query_gecko_profile_options(self): 307 gecko_results = [] 308 # finally, if gecko_profile is set, we add that to the talos options 309 if self.gecko_profile: 310 gecko_results.append("--gecko-profile") 311 for setting in GeckoProfilerSettings: 312 value = getattr(self, setting, None) 313 if value: 314 arg = "--" + setting.replace("_", "-") 315 gecko_results.extend([arg, str(value)]) 316 return gecko_results 317 318 def query_abs_dirs(self): 319 if self.abs_dirs: 320 return self.abs_dirs 321 abs_dirs = super(Talos, self).query_abs_dirs() 322 abs_dirs["abs_blob_upload_dir"] = os.path.join( 323 abs_dirs["abs_work_dir"], "blobber_upload_dir" 324 ) 325 abs_dirs["abs_test_install_dir"] = os.path.join( 326 abs_dirs["abs_work_dir"], "tests" 327 ) 328 self.abs_dirs = abs_dirs 329 return self.abs_dirs 330 331 def query_talos_json_config(self): 332 """Return the talos json config.""" 333 if self.talos_json_config: 334 return self.talos_json_config 335 if not self.talos_json: 336 self.talos_json = os.path.join(self.talos_path, "talos.json") 337 self.talos_json_config = parse_config_file(self.talos_json) 338 self.info(pprint.pformat(self.talos_json_config)) 339 return self.talos_json_config 340 341 def make_talos_domain(self, host): 342 return host + "-talos" 343 344 def split_path(self, path): 345 result = [] 346 while True: 347 path, folder = os.path.split(path) 348 if folder: 349 result.append(folder) 350 continue 351 elif path: 352 result.append(path) 353 break 354 355 result.reverse() 356 return result 357 358 def merge_paths(self, lhs, rhs): 359 backtracks = 0 360 for subdir in rhs: 361 if subdir == "..": 362 backtracks += 1 363 else: 364 break 365 return lhs[:-backtracks] + rhs[backtracks:] 366 367 def replace_relative_iframe_paths(self, directory, filename): 368 """This will find iframes with relative paths and replace them with 369 absolute paths containing domains derived from the original source's 370 domain. This helps us better simulate real-world cases for fission 371 """ 372 if not filename.endswith(".html"): 373 return 374 375 directory_pieces = self.split_path(directory) 376 while directory_pieces and directory_pieces[0] != "fis": 377 directory_pieces = directory_pieces[1:] 378 path = os.path.join(directory, filename) 379 380 # XXX: ugh, is there a better way to account for multiple encodings than just 381 # trying each of them? 382 encodings = ["utf-8", "latin-1"] 383 iframe_pattern = re.compile(r'(iframe.*")(\.\./.*\.html)"') 384 for encoding in encodings: 385 try: 386 with io.open(path, "r", encoding=encoding) as f: 387 content = f.read() 388 389 def replace_iframe_src(match): 390 src = match.group(2) 391 split = self.split_path(src) 392 merged = self.merge_paths(directory_pieces, split) 393 host = merged[3] 394 site_origin_hash = self.make_talos_domain(host) 395 new_url = 'http://%s/%s"' % ( 396 site_origin_hash, 397 "/".join(merged), # pylint --py3k: W1649 398 ) 399 self.info( 400 "Replacing %s with %s in iframe inside %s" 401 % (match.group(2), new_url, path) 402 ) 403 return match.group(1) + new_url 404 405 content = re.sub(iframe_pattern, replace_iframe_src, content) 406 with io.open(path, "w", encoding=encoding) as f: 407 f.write(content) 408 break 409 except UnicodeDecodeError: 410 pass 411 412 def query_pagesets_name(self): 413 """Certain suites require external pagesets to be downloaded and 414 extracted. 415 """ 416 if self.pagesets_name: 417 return self.pagesets_name 418 if self.query_talos_json_config() and self.suite is not None: 419 self.pagesets_name = self.talos_json_config["suites"][self.suite].get( 420 "pagesets_name" 421 ) 422 self.pagesets_name_manifest = "tp5n-pageset.manifest" 423 return self.pagesets_name 424 425 def query_benchmark_zip(self): 426 """Certain suites require external benchmarks to be downloaded and 427 extracted. 428 """ 429 if self.benchmark_zip: 430 return self.benchmark_zip 431 if self.query_talos_json_config() and self.suite is not None: 432 self.benchmark_zip = self.talos_json_config["suites"][self.suite].get( 433 "benchmark_zip" 434 ) 435 self.benchmark_zip_manifest = "jetstream-benchmark.manifest" 436 return self.benchmark_zip 437 438 def query_webextensions_zip(self): 439 """Certain suites require external WebExtension sets to be downloaded and 440 extracted. 441 """ 442 if self.webextensions_zip: 443 return self.webextensions_zip 444 if self.query_talos_json_config() and self.suite is not None: 445 self.webextensions_zip = self.talos_json_config["suites"][self.suite].get( 446 "webextensions_zip" 447 ) 448 self.webextensions_zip_manifest = "webextensions.manifest" 449 return self.webextensions_zip 450 451 def get_suite_from_test(self): 452 """Retrieve the talos suite name from a given talos test name.""" 453 # running locally, single test name provided instead of suite; go through tests and 454 # find suite name 455 suite_name = None 456 if self.query_talos_json_config(): 457 if "-a" in self.config["talos_extra_options"]: 458 test_name_index = self.config["talos_extra_options"].index("-a") + 1 459 if "--activeTests" in self.config["talos_extra_options"]: 460 test_name_index = ( 461 self.config["talos_extra_options"].index("--activeTests") + 1 462 ) 463 if test_name_index < len(self.config["talos_extra_options"]): 464 test_name = self.config["talos_extra_options"][test_name_index] 465 for talos_suite in self.talos_json_config["suites"]: 466 if test_name in self.talos_json_config["suites"][talos_suite].get( 467 "tests" 468 ): 469 suite_name = talos_suite 470 if not suite_name: 471 # no suite found to contain the specified test, error out 472 self.fatal("Test name is missing or invalid") 473 else: 474 self.fatal("Talos json config not found, cannot verify suite") 475 return suite_name 476 477 def validate_suite(self): 478 """Ensure suite name is a valid talos suite.""" 479 if self.query_talos_json_config() and self.suite is not None: 480 if self.suite not in self.talos_json_config.get("suites"): 481 self.fatal( 482 "Suite '%s' is not valid (not found in talos json config)" 483 % self.suite 484 ) 485 486 def talos_options(self, args=None, **kw): 487 """return options to talos""" 488 # binary path 489 binary_path = self.binary_path or self.config.get("binary_path") 490 if not binary_path: 491 msg = """Talos requires a path to the binary. You can specify binary_path or add 492 download-and-extract to your action list.""" 493 self.fatal(msg) 494 495 # talos options 496 options = [] 497 # talos can't gather data if the process name ends with '.exe' 498 if binary_path.endswith(".exe"): 499 binary_path = binary_path[:-4] 500 # options overwritten from **kw 501 kw_options = {"executablePath": binary_path} 502 if "suite" in self.config: 503 kw_options["suite"] = self.config["suite"] 504 if self.config.get("title"): 505 kw_options["title"] = self.config["title"] 506 if self.symbols_path: 507 kw_options["symbolsPath"] = self.symbols_path 508 509 kw_options.update(kw) 510 # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg' 511 tests = kw_options.get("activeTests") 512 if tests and not isinstance(tests, six.string_types): 513 tests = ":".join(tests) # Talos expects this format 514 kw_options["activeTests"] = tests 515 for key, value in kw_options.items(): 516 options.extend(["--%s" % key, value]) 517 # configure profiling options 518 options.extend(self.query_gecko_profile_options()) 519 # extra arguments 520 if args is not None: 521 options += args 522 if "talos_extra_options" in self.config: 523 options += self.config["talos_extra_options"] 524 if self.config.get("code_coverage", False): 525 options.extend(["--code-coverage"]) 526 if self.config["extra_prefs"]: 527 options.extend( 528 ["--setpref={}".format(p) for p in self.config["extra_prefs"]] 529 ) 530 # disabling fission can come from the --disable-fission cmd line argument; or in CI 531 # it comes from a taskcluster transform which adds a --setpref for fission.autostart 532 if (not self.config["fission"]) or "fission.autostart=false" in self.config[ 533 "extra_prefs" 534 ]: 535 options.extend(["--disable-fission"]) 536 537 return options 538 539 def populate_webroot(self): 540 """Populate the production test machines' webroots""" 541 self.talos_path = os.path.join( 542 self.query_abs_dirs()["abs_test_install_dir"], "talos" 543 ) 544 545 # need to determine if talos pageset is required to be downloaded 546 if self.config.get("run_local") and "talos_extra_options" in self.config: 547 # talos initiated locally, get and verify test/suite from cmd line 548 self.talos_path = os.path.dirname(self.talos_json) 549 if ( 550 "-a" in self.config["talos_extra_options"] 551 or "--activeTests" in self.config["talos_extra_options"] 552 ): 553 # test name (-a or --activeTests) specified, find out what suite it is a part of 554 self.suite = self.get_suite_from_test() 555 elif "--suite" in self.config["talos_extra_options"]: 556 # --suite specified, get suite from cmd line and ensure is valid 557 suite_name_index = ( 558 self.config["talos_extra_options"].index("--suite") + 1 559 ) 560 if suite_name_index < len(self.config["talos_extra_options"]): 561 self.suite = self.config["talos_extra_options"][suite_name_index] 562 self.validate_suite() 563 else: 564 self.fatal("Suite name not provided") 565 else: 566 # talos initiated in production via mozharness 567 self.suite = self.config["suite"] 568 569 tooltool_artifacts = [] 570 src_talos_pageset_dest = os.path.join(self.talos_path, "talos", "tests") 571 # unfortunately this path has to be short and can't be descriptive, because 572 # on Windows we tend to already push the boundaries of the max path length 573 # constraint. This will contain the tp5 pageset, but adjusted to have 574 # absolute URLs on iframes for the purposes of better modeling things for 575 # fission. 576 src_talos_pageset_multidomain_dest = os.path.join( 577 self.talos_path, "talos", "fis" 578 ) 579 webextension_dest = os.path.join(self.talos_path, "talos", "webextensions") 580 581 if self.query_pagesets_name(): 582 tooltool_artifacts.append( 583 { 584 "name": self.pagesets_name, 585 "manifest": self.pagesets_name_manifest, 586 "dest": src_talos_pageset_dest, 587 } 588 ) 589 tooltool_artifacts.append( 590 { 591 "name": self.pagesets_name, 592 "manifest": self.pagesets_name_manifest, 593 "dest": src_talos_pageset_multidomain_dest, 594 "postprocess": self.replace_relative_iframe_paths, 595 } 596 ) 597 598 if self.query_benchmark_zip(): 599 tooltool_artifacts.append( 600 { 601 "name": self.benchmark_zip, 602 "manifest": self.benchmark_zip_manifest, 603 "dest": src_talos_pageset_dest, 604 } 605 ) 606 607 if self.query_webextensions_zip(): 608 tooltool_artifacts.append( 609 { 610 "name": self.webextensions_zip, 611 "manifest": self.webextensions_zip_manifest, 612 "dest": webextension_dest, 613 } 614 ) 615 616 # now that have the suite name, check if artifact is required, if so download it 617 # the --no-download option will override this 618 for artifact in tooltool_artifacts: 619 if "--no-download" not in self.config.get("talos_extra_options", []): 620 self.info("Downloading %s with tooltool..." % artifact) 621 622 archive = os.path.join(artifact["dest"], artifact["name"]) 623 output_dir_path = re.sub(r"\.zip$", "", archive) 624 if not os.path.exists(archive): 625 manifest_file = os.path.join(self.talos_path, artifact["manifest"]) 626 self.tooltool_fetch( 627 manifest_file, 628 output_dir=artifact["dest"], 629 cache=self.config.get("tooltool_cache"), 630 ) 631 unzip = self.query_exe("unzip") 632 unzip_cmd = [unzip, "-q", "-o", archive, "-d", artifact["dest"]] 633 self.run_command(unzip_cmd, halt_on_failure=True) 634 635 if "postprocess" in artifact: 636 for subdir, dirs, files in os.walk(output_dir_path): 637 for file in files: 638 artifact["postprocess"](subdir, file) 639 else: 640 self.info("%s already available" % artifact) 641 642 else: 643 self.info( 644 "Not downloading %s because the no-download option was specified" 645 % artifact 646 ) 647 648 # if running webkit tests locally, need to copy webkit source into talos/tests 649 if self.config.get("run_local") and ( 650 "stylebench" in self.suite or "motionmark" in self.suite 651 ): 652 self.get_webkit_source() 653 654 def get_webkit_source(self): 655 # in production the build system auto copies webkit source into place; 656 # but when run locally we need to do this manually, so that talos can find it 657 src = os.path.join(self.repo_path, "third_party", "webkit", "PerformanceTests") 658 dest = os.path.join( 659 self.talos_path, "talos", "tests", "webkit", "PerformanceTests" 660 ) 661 662 if os.path.exists(dest): 663 shutil.rmtree(dest) 664 665 self.info("Copying webkit benchmarks from %s to %s" % (src, dest)) 666 try: 667 shutil.copytree(src, dest) 668 except Exception: 669 self.critical("Error copying webkit benchmarks from %s to %s" % (src, dest)) 670 671 # Action methods. {{{1 672 # clobber defined in BaseScript 673 674 def download_and_extract(self, extract_dirs=None, suite_categories=None): 675 return super(Talos, self).download_and_extract( 676 suite_categories=["common", "talos"] 677 ) 678 679 def create_virtualenv(self, **kwargs): 680 """VirtualenvMixin.create_virtualenv() assuemes we're using 681 self.config['virtualenv_modules']. Since we are installing 682 talos from its source, we have to wrap that method here.""" 683 # if virtualenv already exists, just add to path and don't re-install, need it 684 # in path so can import jsonschema later when validating output for perfherder 685 _virtualenv_path = self.config.get("virtualenv_path") 686 687 if self.run_local and os.path.exists(_virtualenv_path): 688 self.info("Virtualenv already exists, skipping creation") 689 _python_interp = self.config.get("exes")["python"] 690 691 if "win" in self.platform_name(): 692 _path = os.path.join(_virtualenv_path, "Lib", "site-packages") 693 else: 694 _path = os.path.join( 695 _virtualenv_path, 696 "lib", 697 os.path.basename(_python_interp), 698 "site-packages", 699 ) 700 701 sys.path.append(_path) 702 return 703 704 # virtualenv doesn't already exist so create it 705 # install mozbase first, so we use in-tree versions 706 if not self.run_local: 707 mozbase_requirements = os.path.join( 708 self.query_abs_dirs()["abs_test_install_dir"], 709 "config", 710 "mozbase_requirements.txt", 711 ) 712 else: 713 mozbase_requirements = os.path.join( 714 os.path.dirname(self.talos_path), 715 "config", 716 "mozbase_source_requirements.txt", 717 ) 718 self.register_virtualenv_module( 719 requirements=[mozbase_requirements], 720 two_pass=True, 721 editable=True, 722 ) 723 super(Talos, self).create_virtualenv() 724 # talos in harness requires what else is 725 # listed in talos requirements.txt file. 726 self.install_module( 727 requirements=[os.path.join(self.talos_path, "requirements.txt")] 728 ) 729 730 def _validate_treeherder_data(self, parser): 731 # late import is required, because install is done in create_virtualenv 732 import jsonschema 733 734 if len(parser.found_perf_data) != 1: 735 self.critical( 736 "PERFHERDER_DATA was seen %d times, expected 1." 737 % len(parser.found_perf_data) 738 ) 739 parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) 740 return 741 742 schema_path = os.path.join( 743 external_tools_path, "performance-artifact-schema.json" 744 ) 745 self.info("Validating PERFHERDER_DATA against %s" % schema_path) 746 try: 747 with open(schema_path) as f: 748 schema = json.load(f) 749 data = json.loads(parser.found_perf_data[0]) 750 jsonschema.validate(data, schema) 751 except Exception: 752 self.exception("Error while validating PERFHERDER_DATA") 753 parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) 754 755 def _artifact_perf_data(self, parser, dest): 756 src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "local.json") 757 try: 758 shutil.copyfile(src, dest) 759 except Exception: 760 self.critical("Error copying results %s to upload dir %s" % (src, dest)) 761 parser.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_FAILURE) 762 763 def run_tests(self, args=None, **kw): 764 """run Talos tests""" 765 766 # get talos options 767 options = self.talos_options(args=args, **kw) 768 769 # XXX temporary python version check 770 python = self.query_python_path() 771 self.run_command([python, "--version"]) 772 parser = TalosOutputParser( 773 config=self.config, log_obj=self.log_obj, error_list=TalosErrorList 774 ) 775 env = {} 776 env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] 777 if not self.run_local: 778 env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() 779 env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] 780 env["RUST_BACKTRACE"] = "full" 781 if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): 782 self.mkdir_p(env["MOZ_UPLOAD_DIR"]) 783 env = self.query_env(partial_env=env, log_level=INFO) 784 # adjust PYTHONPATH to be able to use talos as a python package 785 if "PYTHONPATH" in env: 786 env["PYTHONPATH"] = self.talos_path + os.pathsep + env["PYTHONPATH"] 787 else: 788 env["PYTHONPATH"] = self.talos_path 789 790 if self.repo_path is not None: 791 env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path 792 if self.obj_path is not None: 793 env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path 794 795 # TODO: consider getting rid of this as we should be default to stylo now 796 env["STYLO_FORCE_ENABLED"] = "1" 797 798 # sets a timeout for how long talos should run without output 799 output_timeout = self.config.get("talos_output_timeout", 3600) 800 # run talos tests 801 run_tests = os.path.join(self.talos_path, "talos", "run_tests.py") 802 803 mozlog_opts = ["--log-tbpl-level=debug"] 804 if not self.run_local and "suite" in self.config: 805 fname_pattern = "%s_%%s.log" % self.config["suite"] 806 mozlog_opts.append( 807 "--log-errorsummary=%s" 808 % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary") 809 ) 810 mozlog_opts.append( 811 "--log-raw=%s" 812 % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "raw") 813 ) 814 815 def launch_in_debug_mode(cmdline): 816 cmdline = set(cmdline) 817 debug_opts = {"--debug", "--debugger", "--debugger_args"} 818 819 return bool(debug_opts.intersection(cmdline)) 820 821 command = [python, run_tests] + options + mozlog_opts 822 if launch_in_debug_mode(command): 823 talos_process = subprocess.Popen( 824 command, cwd=self.workdir, env=env, bufsize=0 825 ) 826 talos_process.wait() 827 else: 828 self.return_code = self.run_command( 829 command, 830 cwd=self.workdir, 831 output_timeout=output_timeout, 832 output_parser=parser, 833 env=env, 834 ) 835 if parser.minidump_output: 836 self.info("Looking at the minidump files for debugging purposes...") 837 for item in parser.minidump_output: 838 self.run_command(["ls", "-l", item]) 839 840 if self.return_code not in [0]: 841 # update the worst log level and tbpl status 842 log_level = ERROR 843 tbpl_level = TBPL_FAILURE 844 if self.return_code == 1: 845 log_level = WARNING 846 tbpl_level = TBPL_WARNING 847 if self.return_code == 4: 848 log_level = WARNING 849 tbpl_level = TBPL_RETRY 850 851 parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level) 852 elif "--no-upload-results" not in options: 853 if not self.gecko_profile: 854 self._validate_treeherder_data(parser) 855 if not self.run_local: 856 # copy results to upload dir so they are included as an artifact 857 dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json") 858 self._artifact_perf_data(parser, dest) 859 860 self.record_status(parser.worst_tbpl_status, level=parser.worst_log_level) 861