1import argparse 2import ast 3import os 4import sys 5from collections import OrderedDict 6from distutils.spawn import find_executable 7 8import config 9import wpttest 10import formatters 11 12 13def abs_path(path): 14 return os.path.abspath(os.path.expanduser(path)) 15 16 17def url_or_path(path): 18 import urlparse 19 20 parsed = urlparse.urlparse(path) 21 if len(parsed.scheme) > 2: 22 return path 23 else: 24 return abs_path(path) 25 26 27def require_arg(kwargs, name, value_func=None): 28 if value_func is None: 29 value_func = lambda x: x is not None 30 31 if name not in kwargs or not value_func(kwargs[name]): 32 print >> sys.stderr, "Missing required argument %s" % name 33 sys.exit(1) 34 35 36def create_parser(product_choices=None): 37 from mozlog import commandline 38 39 import products 40 41 if product_choices is None: 42 config_data = config.load() 43 product_choices = products.products_enabled(config_data) 44 45 parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""", 46 usage="""%(prog)s [OPTION]... [TEST]... 47 48TEST is either the full path to a test file to run, or the URL of a test excluding 49scheme host and port.""") 50 parser.add_argument("--manifest-update", action="store_true", default=None, 51 help="Regenerate the test manifest.") 52 parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update", 53 help="Prevent regeneration of the test manifest.") 54 parser.add_argument("--manifest-download", action="store_true", default=None, 55 help="Attempt to download a preexisting manifest when updating.") 56 57 parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, 58 help="Multiplier relative to standard test timeout to use") 59 parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, 60 help="Split run into groups by directories. With a parameter," 61 "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" 62 "directory") 63 parser.add_argument("--processes", action="store", type=int, default=None, 64 help="Number of simultaneous processes to use") 65 66 parser.add_argument("--no-capture-stdio", action="store_true", default=False, 67 help="Don't capture stdio and write to logging") 68 69 mode_group = parser.add_argument_group("Mode") 70 mode_group.add_argument("--list-test-groups", action="store_true", 71 default=False, 72 help="List the top level directories containing tests that will run.") 73 mode_group.add_argument("--list-disabled", action="store_true", 74 default=False, 75 help="List the tests that are disabled on the current platform") 76 mode_group.add_argument("--list-tests", action="store_true", 77 default=False, 78 help="List all tests that will run") 79 mode_group.add_argument("--verify", action="store_true", 80 default=False, 81 help="Run a stability check on the selected tests") 82 mode_group.add_argument("--verify-log-full", action="store_true", 83 default=False, 84 help="Output per-iteration test results when running verify") 85 86 test_selection_group = parser.add_argument_group("Test Selection") 87 test_selection_group.add_argument("--test-types", action="store", 88 nargs="*", default=wpttest.enabled_tests, 89 choices=wpttest.enabled_tests, 90 help="Test types to run") 91 test_selection_group.add_argument("--include", action="append", 92 help="URL prefix to include") 93 test_selection_group.add_argument("--exclude", action="append", 94 help="URL prefix to exclude") 95 test_selection_group.add_argument("--include-manifest", type=abs_path, 96 help="Path to manifest listing tests to include") 97 test_selection_group.add_argument("--skip-timeout", action="store_true", 98 help="Skip tests that are expected to time out") 99 test_selection_group.add_argument("--tag", action="append", dest="tags", 100 help="Labels applied to tests to include in the run. " 101 "Labels starting dir: are equivalent to top-level directories.") 102 103 debugging_group = parser.add_argument_group("Debugging") 104 debugging_group.add_argument('--debugger', const="__default__", nargs="?", 105 help="run under a debugger, e.g. gdb or valgrind") 106 debugging_group.add_argument('--debugger-args', help="arguments to the debugger") 107 debugging_group.add_argument("--rerun", action="store", type=int, default=1, 108 help="Number of times to re run each test without restarts") 109 debugging_group.add_argument("--repeat", action="store", type=int, default=1, 110 help="Number of times to run the tests, restarting between each run") 111 debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None, 112 help="Run tests in a loop until one returns an unexpected result") 113 debugging_group.add_argument('--pause-after-test', action="store_true", default=None, 114 help="Halt the test runner after each test (this happens by default if only a single test is run)") 115 debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", 116 help="Don't halt the test runner irrespective of the number of tests run") 117 118 debugging_group.add_argument('--pause-on-unexpected', action="store_true", 119 help="Halt the test runner when an unexpected result is encountered") 120 debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected", 121 default=True, action="store_false", 122 help="Don't restart on an unexpected result") 123 124 debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, 125 help="Path or url to symbols file used to analyse crash minidumps.") 126 debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, 127 help="Path to stackwalker program used to analyse minidumps.") 128 129 debugging_group.add_argument("--pdb", action="store_true", 130 help="Drop into pdb on python exception") 131 132 config_group = parser.add_argument_group("Configuration") 133 config_group.add_argument("--binary", action="store", 134 type=abs_path, help="Binary to run tests against") 135 config_group.add_argument('--binary-arg', 136 default=[], action="append", dest="binary_args", 137 help="Extra argument for the binary") 138 config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", 139 type=abs_path, help="WebDriver server binary to use") 140 config_group.add_argument('--webdriver-arg', 141 default=[], action="append", dest="webdriver_args", 142 help="Extra argument for the WebDriver binary") 143 144 config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", 145 help="Path to root directory containing test metadata"), 146 config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root", 147 help="Path to root directory containing test files"), 148 config_group.add_argument("--run-info", action="store", type=abs_path, 149 help="Path to directory containing extra json files to add to run info") 150 config_group.add_argument("--product", action="store", choices=product_choices, 151 default=None, help="Browser against which to run tests") 152 config_group.add_argument("--config", action="store", type=abs_path, dest="config", 153 help="Path to config file") 154 config_group.add_argument("--install-fonts", action="store_true", 155 default=None, 156 help="Allow the wptrunner to install fonts on your system") 157 config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir", 158 help="Path to local font installation directory", default=None) 159 160 build_type = parser.add_mutually_exclusive_group() 161 build_type.add_argument("--debug-build", dest="debug", action="store_true", 162 default=None, 163 help="Build is a debug build (overrides any mozinfo file)") 164 build_type.add_argument("--release-build", dest="debug", action="store_false", 165 default=None, 166 help="Build is a release (overrides any mozinfo file)") 167 168 169 chunking_group = parser.add_argument_group("Test Chunking") 170 chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, 171 help="Total number of chunks to use") 172 chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, 173 help="Chunk number to run") 174 chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"], 175 default=None, help="Chunking type to use") 176 177 ssl_group = parser.add_argument_group("SSL/TLS") 178 ssl_group.add_argument("--ssl-type", action="store", default=None, 179 choices=["openssl", "pregenerated", "none"], 180 help="Type of ssl support to enable (running without ssl may lead to spurious errors)") 181 182 ssl_group.add_argument("--openssl-binary", action="store", 183 help="Path to openssl binary", default="openssl") 184 ssl_group.add_argument("--certutil-binary", action="store", 185 help="Path to certutil binary for use with Firefox + ssl") 186 187 ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, 188 help="Path to ca certificate when using pregenerated ssl certificates") 189 ssl_group.add_argument("--host-key-path", action="store", type=abs_path, 190 help="Path to host private key when using pregenerated ssl certificates") 191 ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, 192 help="Path to host certificate when using pregenerated ssl certificates") 193 194 gecko_group = parser.add_argument_group("Gecko-specific") 195 gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, 196 help="Path to the folder containing browser prefs") 197 gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, 198 help="Run tests without electrolysis preferences") 199 gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store", 200 help="Path to directory containing assertion stack fixing scripts") 201 gecko_group.add_argument("--setpref", dest="extra_prefs", action='append', 202 default=[], metavar="PREF=VALUE", 203 help="Defines an extra user preference (overrides those in prefs_root)") 204 gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true", 205 help="Enable leak checking") 206 gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1, 207 help="Number of parallel threads to use for stylo") 208 gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true", 209 default=None, help="Enable reftest runner implemented inside Marionette") 210 gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false", 211 help="Disable reftest runner implemented inside Marionette") 212 gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store", 213 choices=["always", "fail", "unexpected"], default="unexpected", 214 help="With --reftest-internal, when to take a screenshot") 215 gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store", 216 nargs="?", const=0xFFFFFFFF, type=int, 217 help="Enable chaos mode with the specified feature flag " 218 "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for " 219 "details). If no value is supplied, all features are activated") 220 221 servo_group = parser.add_argument_group("Servo-specific") 222 servo_group.add_argument("--user-stylesheet", 223 default=[], action="append", dest="user_stylesheets", 224 help="Inject a user CSS stylesheet into every test.") 225 226 sauce_group = parser.add_argument_group("Sauce Labs-specific") 227 sauce_group.add_argument("--sauce-browser", dest="sauce_browser", 228 help="Sauce Labs browser name") 229 sauce_group.add_argument("--sauce-platform", dest="sauce_platform", 230 help="Sauce Labs OS platform") 231 sauce_group.add_argument("--sauce-version", dest="sauce_version", 232 help="Sauce Labs browser version") 233 sauce_group.add_argument("--sauce-build", dest="sauce_build", 234 help="Sauce Labs build identifier") 235 sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*", 236 help="Sauce Labs identifying tag", default=[]) 237 sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id", 238 help="Sauce Connect tunnel identifier") 239 sauce_group.add_argument("--sauce-user", dest="sauce_user", 240 help="Sauce Labs user name") 241 sauce_group.add_argument("--sauce-key", dest="sauce_key", 242 default=os.environ.get("SAUCE_ACCESS_KEY"), 243 help="Sauce Labs access key") 244 sauce_group.add_argument("--sauce-connect-binary", 245 dest="sauce_connect_binary", 246 help="Path to Sauce Connect binary") 247 248 parser.add_argument("test_list", nargs="*", 249 help="List of URLs for tests to run, or paths including tests to run. " 250 "(equivalent to --include)") 251 252 commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format") 253 254 commandline.add_logging_group(parser) 255 return parser 256 257 258def set_from_config(kwargs): 259 if kwargs["config"] is None: 260 config_path = config.path() 261 else: 262 config_path = kwargs["config"] 263 264 kwargs["config_path"] = config_path 265 266 kwargs["config"] = config.read(kwargs["config_path"]) 267 268 keys = {"paths": [("prefs", "prefs_root", True), 269 ("run_info", "run_info", True)], 270 "web-platform-tests": [("remote_url", "remote_url", False), 271 ("branch", "branch", False), 272 ("sync_path", "sync_path", True)], 273 "SSL": [("openssl_binary", "openssl_binary", True), 274 ("certutil_binary", "certutil_binary", True), 275 ("ca_cert_path", "ca_cert_path", True), 276 ("host_cert_path", "host_cert_path", True), 277 ("host_key_path", "host_key_path", True)]} 278 279 for section, values in keys.iteritems(): 280 for config_value, kw_value, is_path in values: 281 if kw_value in kwargs and kwargs[kw_value] is None: 282 if not is_path: 283 new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value) 284 else: 285 new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value) 286 kwargs[kw_value] = new_value 287 288 kwargs["test_paths"] = get_test_paths(kwargs["config"]) 289 290 if kwargs["tests_root"]: 291 if "/" not in kwargs["test_paths"]: 292 kwargs["test_paths"]["/"] = {} 293 kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"] 294 295 if kwargs["metadata_root"]: 296 if "/" not in kwargs["test_paths"]: 297 kwargs["test_paths"]["/"] = {} 298 kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"] 299 300 kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests") 301 302 303def get_test_paths(config): 304 # Set up test_paths 305 test_paths = OrderedDict() 306 307 for section in config.iterkeys(): 308 if section.startswith("manifest:"): 309 manifest_opts = config.get(section) 310 url_base = manifest_opts.get("url_base", "/") 311 test_paths[url_base] = { 312 "tests_path": manifest_opts.get_path("tests"), 313 "metadata_path": manifest_opts.get_path("metadata")} 314 315 return test_paths 316 317 318def exe_path(name): 319 if name is None: 320 return 321 322 path = find_executable(name) 323 if path and os.access(path, os.X_OK): 324 return path 325 else: 326 return None 327 328 329def check_args(kwargs): 330 set_from_config(kwargs) 331 332 for test_paths in kwargs["test_paths"].itervalues(): 333 if not ("tests_path" in test_paths and 334 "metadata_path" in test_paths): 335 print "Fatal: must specify both a test path and metadata path" 336 sys.exit(1) 337 for key, path in test_paths.iteritems(): 338 name = key.split("_", 1)[0] 339 340 if not os.path.exists(path): 341 print "Fatal: %s path %s does not exist" % (name, path) 342 sys.exit(1) 343 344 if not os.path.isdir(path): 345 print "Fatal: %s path %s is not a directory" % (name, path) 346 sys.exit(1) 347 348 if kwargs["product"] is None: 349 kwargs["product"] = "firefox" 350 351 if "sauce" in kwargs["product"]: 352 kwargs["pause_after_test"] = False 353 354 if kwargs["test_list"]: 355 if kwargs["include"] is not None: 356 kwargs["include"].extend(kwargs["test_list"]) 357 else: 358 kwargs["include"] = kwargs["test_list"] 359 360 if kwargs["run_info"] is None: 361 kwargs["run_info"] = kwargs["config_path"] 362 363 if kwargs["this_chunk"] > 1: 364 require_arg(kwargs, "total_chunks", lambda x: x >= kwargs["this_chunk"]) 365 366 if kwargs["chunk_type"] is None: 367 if kwargs["total_chunks"] > 1: 368 kwargs["chunk_type"] = "dir_hash" 369 else: 370 kwargs["chunk_type"] = "none" 371 372 if kwargs["processes"] is None: 373 kwargs["processes"] = 1 374 375 if kwargs["debugger"] is not None: 376 import mozdebug 377 if kwargs["debugger"] == "__default__": 378 kwargs["debugger"] = mozdebug.get_default_debugger_name() 379 debug_info = mozdebug.get_debugger_info(kwargs["debugger"], 380 kwargs["debugger_args"]) 381 if debug_info and debug_info.interactive: 382 if kwargs["processes"] != 1: 383 kwargs["processes"] = 1 384 kwargs["no_capture_stdio"] = True 385 kwargs["debug_info"] = debug_info 386 else: 387 kwargs["debug_info"] = None 388 389 if kwargs["binary"] is not None: 390 if not os.path.exists(kwargs["binary"]): 391 print >> sys.stderr, "Binary path %s does not exist" % kwargs["binary"] 392 sys.exit(1) 393 394 if kwargs["ssl_type"] is None: 395 if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]): 396 kwargs["ssl_type"] = "pregenerated" 397 elif exe_path(kwargs["openssl_binary"]) is not None: 398 kwargs["ssl_type"] = "openssl" 399 else: 400 kwargs["ssl_type"] = "none" 401 402 if kwargs["ssl_type"] == "pregenerated": 403 require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x)) 404 require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x)) 405 require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x)) 406 407 elif kwargs["ssl_type"] == "openssl": 408 path = exe_path(kwargs["openssl_binary"]) 409 if path is None: 410 print >> sys.stderr, "openssl-binary argument missing or not a valid executable" 411 sys.exit(1) 412 kwargs["openssl_binary"] = path 413 414 if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox" and kwargs["certutil_binary"]: 415 path = exe_path(kwargs["certutil_binary"]) 416 if path is None: 417 print >> sys.stderr, "certutil-binary argument missing or not a valid executable" 418 sys.exit(1) 419 kwargs["certutil_binary"] = path 420 421 if kwargs['extra_prefs']: 422 missing = any('=' not in prefarg for prefarg in kwargs['extra_prefs']) 423 if missing: 424 print >> sys.stderr, "Preferences via --setpref must be in key=value format" 425 sys.exit(1) 426 kwargs['extra_prefs'] = [tuple(prefarg.split('=', 1)) for prefarg in 427 kwargs['extra_prefs']] 428 429 if kwargs["reftest_internal"] is None: 430 # Default to the internal reftest implementation on Linux and OSX 431 kwargs["reftest_internal"] = sys.platform.startswith("linux") or sys.platform.startswith("darwin") 432 433 return kwargs 434 435 436def check_args_update(kwargs): 437 set_from_config(kwargs) 438 439 if kwargs["product"] is None: 440 kwargs["product"] = "firefox" 441 if kwargs["patch"] is None: 442 kwargs["patch"] = kwargs["sync"] 443 444 for item in kwargs["run_log"]: 445 if os.path.isdir(item): 446 print >> sys.stderr, "Log file %s is a directory" % item 447 sys.exit(1) 448 449 return kwargs 450 451 452def create_parser_update(product_choices=None): 453 from mozlog.structured import commandline 454 455 import products 456 457 if product_choices is None: 458 config_data = config.load() 459 product_choices = products.products_enabled(config_data) 460 461 parser = argparse.ArgumentParser("web-platform-tests-update", 462 description="Update script for web-platform-tests tests.") 463 parser.add_argument("--product", action="store", choices=product_choices, 464 default=None, help="Browser for which metadata is being updated") 465 parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") 466 parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", 467 help="Path to the folder containing test metadata"), 468 parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", 469 help="Path to web-platform-tests"), 470 parser.add_argument("--sync-path", action="store", type=abs_path, 471 help="Path to store git checkout of web-platform-tests during update"), 472 parser.add_argument("--remote_url", action="store", 473 help="URL of web-platfrom-tests repository to sync against"), 474 parser.add_argument("--branch", action="store", type=abs_path, 475 help="Remote branch to sync against") 476 parser.add_argument("--rev", action="store", help="Revision to sync to") 477 parser.add_argument("--patch", action="store_true", dest="patch", default=None, 478 help="Create a VCS commit containing the changes.") 479 parser.add_argument("--no-patch", action="store_false", dest="patch", 480 help="Don't create a VCS commit containing the changes.") 481 parser.add_argument("--sync", dest="sync", action="store_true", default=False, 482 help="Sync the tests with the latest from upstream (implies --patch)") 483 parser.add_argument("--ignore-existing", action="store_true", 484 help="When updating test results only consider results from the logfiles provided, not existing expectations.") 485 parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None, 486 help=("Reason for disabling tests. When updating test results, disable tests that have " 487 "inconsistent results across many runs with the given reason.")) 488 parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script") 489 parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script") 490 parser.add_argument("--exclude", action="store", nargs="*", 491 help="List of glob-style paths to exclude when syncing tests") 492 parser.add_argument("--include", action="store", nargs="*", 493 help="List of glob-style paths to include which would otherwise be excluded when syncing tests") 494 # Should make this required iff run=logfile 495 parser.add_argument("run_log", nargs="*", type=abs_path, 496 help="Log file from run of tests") 497 commandline.add_logging_group(parser) 498 return parser 499 500 501def create_parser_reduce(product_choices=None): 502 parser = create_parser(product_choices) 503 parser.add_argument("target", action="store", help="Test id that is unstable") 504 return parser 505 506 507def parse_args(): 508 parser = create_parser() 509 rv = vars(parser.parse_args()) 510 check_args(rv) 511 return rv 512 513 514def parse_args_update(): 515 parser = create_parser_update() 516 rv = vars(parser.parse_args()) 517 check_args_update(rv) 518 return rv 519 520 521def parse_args_reduce(): 522 parser = create_parser_reduce() 523 rv = vars(parser.parse_args()) 524 check_args(rv) 525 return rv 526