1from __future__ import absolute_import, print_function 2 3import argparse 4import os 5import sys 6from collections import OrderedDict 7import mozinfo 8import mozlog 9 10from six.moves.urllib.parse import urlparse 11 12here = os.path.abspath(os.path.dirname(__file__)) 13 14 15class ReftestArgumentsParser(argparse.ArgumentParser): 16 def __init__(self, **kwargs): 17 super(ReftestArgumentsParser, self).__init__(**kwargs) 18 19 # Try to import a MozbuildObject. Success indicates that we are 20 # running from a source tree. This allows some defaults to be set 21 # from the source tree. 22 try: 23 from mozbuild.base import MozbuildObject 24 25 self.build_obj = MozbuildObject.from_environment(cwd=here) 26 except ImportError: 27 self.build_obj = None 28 29 self.add_argument( 30 "--xre-path", 31 action="store", 32 type=str, 33 dest="xrePath", 34 # individual scripts will set a sane default 35 default=None, 36 help="absolute path to directory containing XRE (probably xulrunner)", 37 ) 38 39 self.add_argument( 40 "--symbols-path", 41 action="store", 42 type=str, 43 dest="symbolsPath", 44 default=None, 45 help="absolute path to directory containing breakpad symbols, " 46 "or the URL of a zip file containing symbols", 47 ) 48 49 self.add_argument( 50 "--debugger", 51 action="store", 52 dest="debugger", 53 help="use the given debugger to launch the application", 54 ) 55 56 self.add_argument( 57 "--debugger-args", 58 action="store", 59 dest="debuggerArgs", 60 help="pass the given args to the debugger _before_ " 61 "the application on the command line", 62 ) 63 64 self.add_argument( 65 "--debugger-interactive", 66 action="store_true", 67 dest="debuggerInteractive", 68 help="prevents the test harness from redirecting " 69 "stdout and stderr for interactive debuggers", 70 ) 71 72 self.add_argument( 73 "--appname", 74 action="store", 75 type=str, 76 dest="app", 77 default=None, 78 help="absolute path to application, overriding default", 79 ) 80 81 self.add_argument( 82 "--extra-profile-file", 83 action="append", 84 dest="extraProfileFiles", 85 default=[], 86 help="copy specified files/dirs to testing profile", 87 ) 88 89 self.add_argument( 90 "--timeout", 91 action="store", 92 dest="timeout", 93 type=int, 94 default=300, # 5 minutes per bug 479518 95 help="reftest will timeout in specified number of seconds. " 96 "[default %(default)s].", 97 ) 98 99 self.add_argument( 100 "--leak-threshold", 101 action="store", 102 type=int, 103 dest="defaultLeakThreshold", 104 default=0, 105 help="fail if the number of bytes leaked in default " 106 "processes through refcounted objects (or bytes " 107 "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " 108 "is greater than the given number", 109 ) 110 111 self.add_argument( 112 "--utility-path", 113 action="store", 114 type=str, 115 dest="utilityPath", 116 default=self.build_obj.bindir if self.build_obj else None, 117 help="absolute path to directory containing utility " 118 "programs (xpcshell, ssltunnel, certutil)", 119 ) 120 121 self.add_argument( 122 "--total-chunks", 123 type=int, 124 dest="totalChunks", 125 help="how many chunks to split the tests up into", 126 ) 127 128 self.add_argument( 129 "--this-chunk", 130 type=int, 131 dest="thisChunk", 132 help="which chunk to run between 1 and --total-chunks", 133 ) 134 135 self.add_argument( 136 "--log-file", 137 action="store", 138 type=str, 139 dest="logFile", 140 default=None, 141 help="file to log output to in addition to stdout", 142 ) 143 144 self.add_argument( 145 "--skip-slow-tests", 146 dest="skipSlowTests", 147 action="store_true", 148 default=False, 149 help="skip tests marked as slow when running", 150 ) 151 152 self.add_argument( 153 "--ignore-window-size", 154 dest="ignoreWindowSize", 155 action="store_true", 156 default=False, 157 help="ignore the window size, which may cause spurious " 158 "failures and passes", 159 ) 160 161 self.add_argument( 162 "--install-extension", 163 action="append", 164 dest="extensionsToInstall", 165 default=[], 166 help="install the specified extension in the testing profile. " 167 "The extension file's name should be <id>.xpi where <id> is " 168 "the extension's id as indicated in its install.rdf. " 169 "An optional path can be specified too.", 170 ) 171 172 self.add_argument( 173 "--marionette", 174 default=None, 175 help="host:port to use when connecting to Marionette", 176 ) 177 178 self.add_argument( 179 "--marionette-socket-timeout", default=None, help=argparse.SUPPRESS 180 ) 181 182 self.add_argument( 183 "--marionette-startup-timeout", default=None, help=argparse.SUPPRESS 184 ) 185 186 self.add_argument( 187 "--setenv", 188 action="append", 189 type=str, 190 default=[], 191 dest="environment", 192 metavar="NAME=VALUE", 193 help="sets the given variable in the application's " "environment", 194 ) 195 196 self.add_argument( 197 "--filter", 198 action="store", 199 type=str, 200 dest="filter", 201 help="specifies a regular expression (as could be passed to the JS " 202 "RegExp constructor) to test against URLs in the reftest manifest; " 203 "only test items that have a matching test URL will be run.", 204 ) 205 206 self.add_argument( 207 "--shuffle", 208 action="store_true", 209 default=False, 210 dest="shuffle", 211 help="run reftests in random order", 212 ) 213 214 self.add_argument( 215 "--run-until-failure", 216 action="store_true", 217 default=False, 218 dest="runUntilFailure", 219 help="stop running on the first failure. Useful for RR recordings.", 220 ) 221 222 self.add_argument( 223 "--repeat", 224 action="store", 225 type=int, 226 default=0, 227 dest="repeat", 228 help="number of times the select test(s) will be executed. Useful for " 229 "finding intermittent failures.", 230 ) 231 232 self.add_argument( 233 "--focus-filter-mode", 234 action="store", 235 type=str, 236 dest="focusFilterMode", 237 default="all", 238 help="filters tests to run by whether they require focus. " 239 "Valid values are `all', `needs-focus', or `non-needs-focus'. " 240 "Defaults to `all'.", 241 ) 242 243 self.add_argument( 244 "--disable-e10s", 245 action="store_false", 246 default=True, 247 dest="e10s", 248 help="disables content processes", 249 ) 250 251 self.add_argument( 252 "--enable-fission", 253 action="store_true", 254 default=False, 255 dest="fission", 256 help="Run tests with fission (site isolation) enabled.", 257 ) 258 259 self.add_argument( 260 "--setpref", 261 action="append", 262 type=str, 263 default=[], 264 dest="extraPrefs", 265 metavar="PREF=VALUE", 266 help="defines an extra user preference", 267 ) 268 269 self.add_argument( 270 "--reftest-extension-path", 271 action="store", 272 dest="reftestExtensionPath", 273 help="Path to the reftest extension", 274 ) 275 276 self.add_argument( 277 "--special-powers-extension-path", 278 action="store", 279 dest="specialPowersExtensionPath", 280 help="Path to the special powers extension", 281 ) 282 283 self.add_argument( 284 "--suite", 285 choices=["reftest", "crashtest", "jstestbrowser"], 286 default=None, 287 help=argparse.SUPPRESS, 288 ) 289 290 self.add_argument( 291 "--cleanup-crashes", 292 action="store_true", 293 dest="cleanupCrashes", 294 default=False, 295 help="Delete pending crash reports before running tests.", 296 ) 297 298 self.add_argument( 299 "--max-retries", 300 type=int, 301 dest="maxRetries", 302 default=4, 303 help="The maximum number of attempts to try and recover from a " 304 "crash before aborting the test run [default 4].", 305 ) 306 307 self.add_argument( 308 "tests", 309 metavar="TEST_PATH", 310 nargs="*", 311 help="Path to test file, manifest file, or directory containing " 312 "tests. For jstestbrowser, the relative path can be either from " 313 "topsrcdir or the staged area " 314 "($OBJDIR/dist/test-stage/jsreftest/tests)", 315 ) 316 317 self.add_argument( 318 "--sandbox-read-whitelist", 319 action="append", 320 dest="sandboxReadWhitelist", 321 default=[], 322 help="Path to add to the sandbox whitelist.", 323 ) 324 325 self.add_argument( 326 "--verify", 327 action="store_true", 328 default=False, 329 help="Run tests in verification mode: Run many times in different " 330 "ways, to see if there are intermittent failures.", 331 ) 332 333 self.add_argument( 334 "--verify-max-time", 335 type=int, 336 default=3600, 337 help="Maximum time, in seconds, to run in --verify mode..", 338 ) 339 340 self.add_argument( 341 "--enable-webrender", 342 action="store_true", 343 dest="enable_webrender", 344 default=False, 345 help="Enable the WebRender compositor in Gecko.", 346 ) 347 348 self.add_argument( 349 "--headless", 350 action="store_true", 351 dest="headless", 352 default=False, 353 help="Run tests in headless mode.", 354 ) 355 356 self.add_argument( 357 "--topsrcdir", 358 action="store", 359 type=str, 360 dest="topsrcdir", 361 default=None, 362 help="Path to source directory", 363 ) 364 365 mozlog.commandline.add_logging_group(self) 366 367 def get_ip(self): 368 import moznetwork 369 370 if os.name != "nt": 371 return moznetwork.get_ip() 372 else: 373 self.error("ERROR: you must specify a --remote-webserver=<ip address>\n") 374 375 def set_default_suite(self, options): 376 manifests = OrderedDict( 377 [ 378 ("reftest.list", "reftest"), 379 ("crashtests.list", "crashtest"), 380 ("jstests.list", "jstestbrowser"), 381 ] 382 ) 383 384 for test_path in options.tests: 385 file_name = os.path.basename(test_path) 386 if file_name in manifests: 387 options.suite = manifests[file_name] 388 return 389 390 for test_path in options.tests: 391 for manifest_file, suite in manifests.iteritems(): 392 if os.path.exists(os.path.join(test_path, manifest_file)): 393 options.suite = suite 394 return 395 396 self.error( 397 "Failed to determine test suite; supply --suite to set this explicitly" 398 ) 399 400 def validate(self, options, reftest): 401 if not options.tests: 402 # Can't just set this in the argument parser because mach will set a default 403 self.error( 404 "Must supply at least one path to a manifest file, " 405 "test directory, or test file to run." 406 ) 407 408 if options.suite is None: 409 self.set_default_suite(options) 410 411 if options.totalChunks is not None and options.thisChunk is None: 412 self.error("thisChunk must be specified when totalChunks is specified") 413 414 if options.totalChunks: 415 if not 1 <= options.thisChunk <= options.totalChunks: 416 self.error("thisChunk must be between 1 and totalChunks") 417 418 if options.fission and not options.e10s: 419 self.error("Fission is not supported without e10s.") 420 421 if options.logFile: 422 options.logFile = reftest.getFullPath(options.logFile) 423 424 if options.xrePath is not None: 425 if not os.access(options.xrePath, os.F_OK): 426 self.error("--xre-path '%s' not found" % options.xrePath) 427 if not os.path.isdir(options.xrePath): 428 self.error("--xre-path '%s' is not a directory" % options.xrePath) 429 options.xrePath = reftest.getFullPath(options.xrePath) 430 431 if options.reftestExtensionPath is None: 432 if self.build_obj is not None: 433 reftestExtensionPath = os.path.join( 434 self.build_obj.distdir, "xpi-stage", "reftest" 435 ) 436 else: 437 reftestExtensionPath = os.path.join(here, "reftest") 438 options.reftestExtensionPath = os.path.normpath(reftestExtensionPath) 439 440 if options.specialPowersExtensionPath is None: 441 if self.build_obj is not None: 442 specialPowersExtensionPath = os.path.join( 443 self.build_obj.distdir, "xpi-stage", "specialpowers" 444 ) 445 else: 446 specialPowersExtensionPath = os.path.join(here, "specialpowers") 447 options.specialPowersExtensionPath = os.path.normpath( 448 specialPowersExtensionPath 449 ) 450 451 options.leakThresholds = { 452 "default": options.defaultLeakThreshold, 453 "tab": options.defaultLeakThreshold, 454 } 455 456 if mozinfo.isWin: 457 if mozinfo.info["bits"] == 32: 458 # See bug 1408554. 459 options.leakThresholds["tab"] = 3000 460 else: 461 # See bug 1404482. 462 options.leakThresholds["tab"] = 100 463 464 if options.topsrcdir is None: 465 if self.build_obj: 466 options.topsrcdir = self.build_obj.topsrcdir 467 else: 468 options.topsrcdir = os.getcwd() 469 470 471class DesktopArgumentsParser(ReftestArgumentsParser): 472 def __init__(self, **kwargs): 473 super(DesktopArgumentsParser, self).__init__(**kwargs) 474 475 self.add_argument( 476 "--run-tests-in-parallel", 477 action="store_true", 478 default=False, 479 dest="runTestsInParallel", 480 help="run tests in parallel if possible", 481 ) 482 483 def _prefs_gpu(self): 484 if mozinfo.os != "win": 485 return ["layers.acceleration.force-enabled=true"] 486 return [] 487 488 def validate(self, options, reftest): 489 super(DesktopArgumentsParser, self).validate(options, reftest) 490 491 if options.runTestsInParallel: 492 if options.logFile is not None: 493 self.error("cannot specify logfile with parallel tests") 494 if options.totalChunks is not None or options.thisChunk is not None: 495 self.error( 496 "cannot specify thisChunk or totalChunks with parallel tests" 497 ) 498 if options.focusFilterMode != "all": 499 self.error("cannot specify focusFilterMode with parallel tests") 500 if options.debugger is not None: 501 self.error("cannot specify a debugger with parallel tests") 502 503 if options.debugger: 504 # valgrind and some debuggers may cause Gecko to start slowly. Make sure 505 # marionette waits long enough to connect. 506 options.marionette_startup_timeout = 900 507 options.marionette_socket_timeout = 540 508 509 if not options.tests: 510 self.error("No test files specified.") 511 512 if options.app is None: 513 if ( 514 self.build_obj 515 and self.build_obj.substs["MOZ_BUILD_APP"] != "mobile/android" 516 ): 517 from mozbuild.base import BinaryNotFoundException 518 519 try: 520 bin_dir = self.build_obj.get_binary_path() 521 except BinaryNotFoundException as e: 522 print("{}\n\n{}\n".format(e, e.help()), file=sys.stderr) 523 sys.exit(1) 524 else: 525 bin_dir = None 526 527 if bin_dir: 528 options.app = bin_dir 529 530 if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: 531 options.symbolsPath = reftest.getFullPath(options.symbolsPath) 532 533 options.utilityPath = reftest.getFullPath(options.utilityPath) 534 535 536class RemoteArgumentsParser(ReftestArgumentsParser): 537 def __init__(self, **kwargs): 538 super(RemoteArgumentsParser, self).__init__() 539 540 # app, xrePath and utilityPath variables are set in main function 541 self.set_defaults( 542 logFile="reftest.log", app="", xrePath="", utilityPath="", localLogName=None 543 ) 544 545 self.add_argument( 546 "--adbpath", 547 action="store", 548 type=str, 549 dest="adb_path", 550 default=None, 551 help="Path to adb binary.", 552 ) 553 554 self.add_argument( 555 "--deviceSerial", 556 action="store", 557 type=str, 558 dest="deviceSerial", 559 help="adb serial number of remote device. This is required " 560 "when more than one device is connected to the host. " 561 "Use 'adb devices' to see connected devices.", 562 ) 563 564 self.add_argument( 565 "--remote-webserver", 566 action="store", 567 type=str, 568 dest="remoteWebServer", 569 help="IP address of the remote web server.", 570 ) 571 572 self.add_argument( 573 "--http-port", 574 action="store", 575 type=str, 576 dest="httpPort", 577 help="http port of the remote web server.", 578 ) 579 580 self.add_argument( 581 "--ssl-port", 582 action="store", 583 type=str, 584 dest="sslPort", 585 help="ssl port of the remote web server.", 586 ) 587 588 self.add_argument( 589 "--remoteTestRoot", 590 action="store", 591 type=str, 592 dest="remoteTestRoot", 593 help="Remote directory to use as test root " 594 "(eg. /data/local/tmp/test_root).", 595 ) 596 597 self.add_argument( 598 "--httpd-path", 599 action="store", 600 type=str, 601 dest="httpdPath", 602 help="Path to the httpd.js file.", 603 ) 604 605 self.add_argument( 606 "--no-install", 607 action="store_true", 608 default=False, 609 help="Skip the installation of the APK.", 610 ) 611 612 def validate_remote(self, options): 613 DEFAULT_HTTP_PORT = 8888 614 DEFAULT_SSL_PORT = 4443 615 616 if options.remoteWebServer is None: 617 options.remoteWebServer = self.get_ip() 618 619 if options.remoteWebServer == "127.0.0.1": 620 self.error( 621 "ERROR: Either you specified the loopback for the remote webserver or ", 622 "your local IP cannot be detected. " 623 "Please provide the local ip in --remote-webserver", 624 ) 625 626 if not options.httpPort: 627 options.httpPort = DEFAULT_HTTP_PORT 628 629 if not options.sslPort: 630 options.sslPort = DEFAULT_SSL_PORT 631 632 if options.xrePath is None: 633 self.error( 634 "ERROR: You must specify the path to the controller xre directory" 635 ) 636 else: 637 # Ensure xrepath is a full path 638 options.xrePath = os.path.abspath(options.xrePath) 639 640 # httpd-path is specified by standard makefile targets and may be specified 641 # on the command line to select a particular version of httpd.js. If not 642 # specified, try to select the one from hostutils.zip, as required in 643 # bug 882932. 644 if not options.httpdPath: 645 options.httpdPath = os.path.join(options.utilityPath, "components") 646 647 return options 648