1import argparse 2import os 3import sys 4from collections import OrderedDict 5from distutils.spawn import find_executable 6from datetime import timedelta 7 8from . import config 9from . import wpttest 10from .formatters import chromium, wptreport, wptscreenshot 11 12def abs_path(path): 13 return os.path.abspath(os.path.expanduser(path)) 14 15 16def url_or_path(path): 17 from urllib.parse import urlparse 18 19 parsed = urlparse(path) 20 if len(parsed.scheme) > 2: 21 return path 22 else: 23 return abs_path(path) 24 25 26def require_arg(kwargs, name, value_func=None): 27 if value_func is None: 28 value_func = lambda x: x is not None 29 30 if name not in kwargs or not value_func(kwargs[name]): 31 print("Missing required argument %s" % name, file=sys.stderr) 32 sys.exit(1) 33 34 35def create_parser(product_choices=None): 36 from mozlog import commandline 37 38 from . import products 39 40 if product_choices is None: 41 config_data = config.load() 42 product_choices = products.products_enabled(config_data) 43 44 parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""", 45 usage="""%(prog)s [OPTION]... [TEST]... 46 47TEST is either the full path to a test file to run, or the URL of a test excluding 48scheme host and port.""") 49 parser.add_argument("--manifest-update", action="store_true", default=None, 50 help="Regenerate the test manifest.") 51 parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update", 52 help="Prevent regeneration of the test manifest.") 53 parser.add_argument("--manifest-download", action="store_true", default=None, 54 help="Attempt to download a preexisting manifest when updating.") 55 parser.add_argument("--no-manifest-download", action="store_false", dest="manifest_download", 56 help="Prevent download of the test manifest.") 57 58 parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, 59 help="Multiplier relative to standard test timeout to use") 60 parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, 61 help="Split run into groups by directories. With a parameter," 62 "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" 63 "directory") 64 parser.add_argument("--processes", action="store", type=int, default=None, 65 help="Number of simultaneous processes to use") 66 67 parser.add_argument("--no-capture-stdio", action="store_true", default=False, 68 help="Don't capture stdio and write to logging") 69 parser.add_argument("--no-fail-on-unexpected", action="store_false", 70 default=True, 71 dest="fail_on_unexpected", 72 help="Exit with status code 0 when test expectations are violated") 73 parser.add_argument("--no-fail-on-unexpected-pass", action="store_false", 74 default=True, 75 dest="fail_on_unexpected_pass", 76 help="Exit with status code 0 when all unexpected results are PASS") 77 78 mode_group = parser.add_argument_group("Mode") 79 mode_group.add_argument("--list-test-groups", action="store_true", 80 default=False, 81 help="List the top level directories containing tests that will run.") 82 mode_group.add_argument("--list-disabled", action="store_true", 83 default=False, 84 help="List the tests that are disabled on the current platform") 85 mode_group.add_argument("--list-tests", action="store_true", 86 default=False, 87 help="List all tests that will run") 88 stability_group = mode_group.add_mutually_exclusive_group() 89 stability_group.add_argument("--verify", action="store_true", 90 default=False, 91 help="Run a stability check on the selected tests") 92 stability_group.add_argument("--stability", action="store_true", 93 default=False, 94 help=argparse.SUPPRESS) 95 mode_group.add_argument("--verify-log-full", action="store_true", 96 default=False, 97 help="Output per-iteration test results when running verify") 98 mode_group.add_argument("--verify-repeat-loop", action="store", 99 default=10, 100 help="Number of iterations for a run that reloads each test without restart.", 101 type=int) 102 mode_group.add_argument("--verify-repeat-restart", action="store", 103 default=5, 104 help="Number of iterations, for a run that restarts the runner between each iteration", 105 type=int) 106 chaos_mode_group = mode_group.add_mutually_exclusive_group() 107 chaos_mode_group.add_argument("--verify-no-chaos-mode", action="store_false", 108 default=True, 109 dest="verify_chaos_mode", 110 help="Disable chaos mode when running on Firefox") 111 chaos_mode_group.add_argument("--verify-chaos-mode", action="store_true", 112 default=True, 113 dest="verify_chaos_mode", 114 help="Enable chaos mode when running on Firefox") 115 mode_group.add_argument("--verify-max-time", action="store", 116 default=None, 117 help="The maximum number of minutes for the job to run", 118 type=lambda x: timedelta(minutes=float(x))) 119 output_results_group = mode_group.add_mutually_exclusive_group() 120 output_results_group.add_argument("--verify-no-output-results", action="store_false", 121 dest="verify_output_results", 122 default=True, 123 help="Prints individuals test results and messages") 124 output_results_group.add_argument("--verify-output-results", action="store_true", 125 dest="verify_output_results", 126 default=True, 127 help="Disable printing individuals test results and messages") 128 129 test_selection_group = parser.add_argument_group("Test Selection") 130 test_selection_group.add_argument("--test-types", action="store", 131 nargs="*", default=wpttest.enabled_tests, 132 choices=wpttest.enabled_tests, 133 help="Test types to run") 134 test_selection_group.add_argument("--include", action="append", 135 help="URL prefix to include") 136 test_selection_group.add_argument("--include-file", action="store", 137 help="A file listing URL prefix for tests") 138 test_selection_group.add_argument("--exclude", action="append", 139 help="URL prefix to exclude") 140 test_selection_group.add_argument("--include-manifest", type=abs_path, 141 help="Path to manifest listing tests to include") 142 test_selection_group.add_argument("--test-groups", dest="test_groups_file", type=abs_path, 143 help="Path to json file containing a mapping {group_name: [test_ids]}") 144 test_selection_group.add_argument("--skip-timeout", action="store_true", 145 help="Skip tests that are expected to time out") 146 test_selection_group.add_argument("--skip-implementation-status", 147 action="append", 148 choices=["not-implementing", "backlog", "implementing"], 149 help="Skip tests that have the given implementation status") 150 # TODO: Remove this when QUIC is enabled by default. 151 test_selection_group.add_argument("--enable-quic", action="store_true", default=False, 152 help="Enable tests that require QUIC server (default: false)") 153 154 test_selection_group.add_argument("--tag", action="append", dest="tags", 155 help="Labels applied to tests to include in the run. " 156 "Labels starting dir: are equivalent to top-level directories.") 157 test_selection_group.add_argument("--default-exclude", action="store_true", 158 default=False, 159 help="Only run the tests explicitly given in arguments. " 160 "No tests will run if the list is empty, and the " 161 "program will exit with status code 0.") 162 163 debugging_group = parser.add_argument_group("Debugging") 164 debugging_group.add_argument('--debugger', const="__default__", nargs="?", 165 help="run under a debugger, e.g. gdb or valgrind") 166 debugging_group.add_argument('--debugger-args', help="arguments to the debugger") 167 debugging_group.add_argument("--rerun", action="store", type=int, default=1, 168 help="Number of times to re run each test without restarts") 169 debugging_group.add_argument("--repeat", action="store", type=int, default=1, 170 help="Number of times to run the tests, restarting between each run") 171 debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None, 172 help="Run tests in a loop until one returns an unexpected result") 173 debugging_group.add_argument('--pause-after-test', action="store_true", default=None, 174 help="Halt the test runner after each test (this happens by default if only a single test is run)") 175 debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", 176 help="Don't halt the test runner irrespective of the number of tests run") 177 debugging_group.add_argument('--debug-test', dest="debug_test", action="store_true", 178 help="Run tests with additional debugging features enabled") 179 180 debugging_group.add_argument('--pause-on-unexpected', action="store_true", 181 help="Halt the test runner when an unexpected result is encountered") 182 debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected", 183 default=True, action="store_false", 184 help="Don't restart on an unexpected result") 185 186 debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, 187 help="Path or url to symbols file used to analyse crash minidumps.") 188 debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, 189 help="Path to stackwalker program used to analyse minidumps.") 190 debugging_group.add_argument("--pdb", action="store_true", 191 help="Drop into pdb on python exception") 192 193 config_group = parser.add_argument_group("Configuration") 194 config_group.add_argument("--binary", action="store", 195 type=abs_path, help="Desktop binary to run tests against") 196 config_group.add_argument('--binary-arg', 197 default=[], action="append", dest="binary_args", 198 help="Extra argument for the binary") 199 config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", 200 type=abs_path, help="WebDriver server binary to use") 201 config_group.add_argument('--webdriver-arg', 202 default=[], action="append", dest="webdriver_args", 203 help="Extra argument for the WebDriver binary") 204 config_group.add_argument("--adb-binary", action="store", 205 help="Path to adb binary to use") 206 config_group.add_argument("--package-name", action="store", 207 help="Android package name to run tests against") 208 config_group.add_argument("--device-serial", action="store", 209 help="Running Android instance to connect to, if not emulator-5554") 210 config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", 211 help="Path to root directory containing test metadata"), 212 config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root", 213 help="Path to root directory containing test files"), 214 config_group.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path", 215 help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)") 216 config_group.add_argument("--run-info", action="store", type=abs_path, 217 help="Path to directory containing extra json files to add to run info") 218 config_group.add_argument("--product", action="store", choices=product_choices, 219 default=None, help="Browser against which to run tests") 220 config_group.add_argument("--browser-version", action="store", 221 default=None, help="Informative string detailing the browser " 222 "release version. This is included in the run_info data.") 223 config_group.add_argument("--browser-channel", action="store", 224 default=None, help="Informative string detailing the browser " 225 "release channel. This is included in the run_info data.") 226 config_group.add_argument("--config", action="store", type=abs_path, dest="config", 227 help="Path to config file") 228 config_group.add_argument("--install-fonts", action="store_true", 229 default=None, 230 help="Install additional system fonts on your system") 231 config_group.add_argument("--no-install-fonts", dest="install_fonts", action="store_false", 232 help="Do not install additional system fonts on your system") 233 config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir", 234 help="Path to local font installation directory", default=None) 235 config_group.add_argument("--headless", action="store_true", 236 help="Run browser in headless mode", default=None) 237 config_group.add_argument("--no-headless", action="store_false", dest="headless", 238 help="Don't run browser in headless mode") 239 config_group.add_argument("--instrument-to-file", action="store", 240 help="Path to write instrumentation logs to") 241 242 build_type = parser.add_mutually_exclusive_group() 243 build_type.add_argument("--debug-build", dest="debug", action="store_true", 244 default=None, 245 help="Build is a debug build (overrides any mozinfo file)") 246 build_type.add_argument("--release-build", dest="debug", action="store_false", 247 default=None, 248 help="Build is a release (overrides any mozinfo file)") 249 250 chunking_group = parser.add_argument_group("Test Chunking") 251 chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, 252 help="Total number of chunks to use") 253 chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, 254 help="Chunk number to run") 255 chunking_group.add_argument("--chunk-type", action="store", choices=["none", "hash", "dir_hash"], 256 default=None, help="Chunking type to use") 257 258 ssl_group = parser.add_argument_group("SSL/TLS") 259 ssl_group.add_argument("--ssl-type", action="store", default=None, 260 choices=["openssl", "pregenerated", "none"], 261 help="Type of ssl support to enable (running without ssl may lead to spurious errors)") 262 263 ssl_group.add_argument("--openssl-binary", action="store", 264 help="Path to openssl binary", default="openssl") 265 ssl_group.add_argument("--certutil-binary", action="store", 266 help="Path to certutil binary for use with Firefox + ssl") 267 268 ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, 269 help="Path to ca certificate when using pregenerated ssl certificates") 270 ssl_group.add_argument("--host-key-path", action="store", type=abs_path, 271 help="Path to host private key when using pregenerated ssl certificates") 272 ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, 273 help="Path to host certificate when using pregenerated ssl certificates") 274 275 gecko_group = parser.add_argument_group("Gecko-specific") 276 gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, 277 help="Path to the folder containing browser prefs") 278 gecko_group.add_argument("--preload-browser", dest="preload_browser", action="store_true", 279 default=None, help="Preload a gecko instance for faster restarts") 280 gecko_group.add_argument("--no-preload-browser", dest="preload_browser", action="store_false", 281 default=None, help="Don't preload a gecko instance for faster restarts") 282 gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, 283 help="Run tests without electrolysis preferences") 284 gecko_group.add_argument("--enable-webrender", dest="enable_webrender", action="store_true", default=None, 285 help="Enable the WebRender compositor in Gecko (defaults to disabled).") 286 gecko_group.add_argument("--no-enable-webrender", dest="enable_webrender", action="store_false", 287 help="Disable the WebRender compositor in Gecko.") 288 gecko_group.add_argument("--enable-fission", dest="enable_fission", action="store_true", default=None, 289 help="Enable fission in Gecko (defaults to disabled).") 290 gecko_group.add_argument("--no-enable-fission", dest="enable_fission", action="store_false", 291 help="Disable fission in Gecko.") 292 gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store", 293 help="Path to directory containing assertion stack fixing scripts") 294 gecko_group.add_argument("--specialpowers-path", action="store", 295 help="Path to specialPowers extension xpi file") 296 gecko_group.add_argument("--setpref", dest="extra_prefs", action='append', 297 default=[], metavar="PREF=VALUE", 298 help="Defines an extra user preference (overrides those in prefs_root)") 299 gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true", default=None, 300 help="Enable leak checking (enabled by default for debug builds, " 301 "silently ignored for opt, mobile)") 302 gecko_group.add_argument("--no-leak-check", dest="leak_check", action="store_false", default=None, 303 help="Disable leak checking") 304 gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1, 305 help="Number of parallel threads to use for stylo") 306 gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true", 307 default=None, help="Enable reftest runner implemented inside Marionette") 308 gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false", 309 help="Disable reftest runner implemented inside Marionette") 310 gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store", 311 choices=["always", "fail", "unexpected"], default=None, 312 help="With --reftest-internal, when to take a screenshot") 313 gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store", 314 nargs="?", const=0xFFFFFFFF, type=int, 315 help="Enable chaos mode with the specified feature flag " 316 "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for " 317 "details). If no value is supplied, all features are activated") 318 319 servo_group = parser.add_argument_group("Servo-specific") 320 servo_group.add_argument("--user-stylesheet", 321 default=[], action="append", dest="user_stylesheets", 322 help="Inject a user CSS stylesheet into every test.") 323 324 chrome_group = parser.add_argument_group("Chrome-specific") 325 chrome_group.add_argument("--enable-mojojs", action="store_true", default=False, 326 help="Enable MojoJS for testing. Note that this flag is usally " 327 "enabled automatically by `wpt run`, if it succeeds in downloading " 328 "the right version of mojojs.zip or if --mojojs-path is specified.") 329 chrome_group.add_argument("--mojojs-path", 330 help="Path to mojojs gen/ directory. If it is not specified, `wpt run` " 331 "will download and extract mojojs.zip into _venv2/mojojs/gen.") 332 chrome_group.add_argument("--enable-swiftshader", action="store_true", default=False, 333 help="Enable SwiftShader for CPU-based 3D graphics. This can be used " 334 "in environments with no hardware GPU available.") 335 336 sauce_group = parser.add_argument_group("Sauce Labs-specific") 337 sauce_group.add_argument("--sauce-browser", dest="sauce_browser", 338 help="Sauce Labs browser name") 339 sauce_group.add_argument("--sauce-platform", dest="sauce_platform", 340 help="Sauce Labs OS platform") 341 sauce_group.add_argument("--sauce-version", dest="sauce_version", 342 help="Sauce Labs browser version") 343 sauce_group.add_argument("--sauce-build", dest="sauce_build", 344 help="Sauce Labs build identifier") 345 sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*", 346 help="Sauce Labs identifying tag", default=[]) 347 sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id", 348 help="Sauce Connect tunnel identifier") 349 sauce_group.add_argument("--sauce-user", dest="sauce_user", 350 help="Sauce Labs user name") 351 sauce_group.add_argument("--sauce-key", dest="sauce_key", 352 default=os.environ.get("SAUCE_ACCESS_KEY"), 353 help="Sauce Labs access key") 354 sauce_group.add_argument("--sauce-connect-binary", 355 dest="sauce_connect_binary", 356 help="Path to Sauce Connect binary") 357 sauce_group.add_argument("--sauce-init-timeout", action="store", 358 type=int, default=30, 359 help="Number of seconds to wait for Sauce " 360 "Connect tunnel to be available before " 361 "aborting") 362 sauce_group.add_argument("--sauce-connect-arg", action="append", 363 default=[], dest="sauce_connect_args", 364 help="Command-line argument to forward to the " 365 "Sauce Connect binary (repeatable)") 366 367 taskcluster_group = parser.add_argument_group("Taskcluster-specific") 368 taskcluster_group.add_argument("--github-checks-text-file", 369 type=str, 370 help="Path to GitHub checks output file") 371 372 webkit_group = parser.add_argument_group("WebKit-specific") 373 webkit_group.add_argument("--webkit-port", dest="webkit_port", 374 help="WebKit port") 375 376 safari_group = parser.add_argument_group("Safari-specific") 377 safari_group.add_argument("--kill-safari", dest="kill_safari", action="store_true", default=False, 378 help="Kill Safari when stopping the browser") 379 380 parser.add_argument("test_list", nargs="*", 381 help="List of URLs for tests to run, or paths including tests to run. " 382 "(equivalent to --include)") 383 384 def screenshot_api_wrapper(formatter, api): 385 formatter.api = api 386 return formatter 387 388 commandline.fmt_options["api"] = (screenshot_api_wrapper, 389 "Cache API (default: %s)" % wptscreenshot.DEFAULT_API, 390 {"wptscreenshot"}, "store") 391 392 commandline.log_formatters["chromium"] = (chromium.ChromiumFormatter, "Chromium Layout Tests format") 393 commandline.log_formatters["wptreport"] = (wptreport.WptreportFormatter, "wptreport format") 394 commandline.log_formatters["wptscreenshot"] = (wptscreenshot.WptscreenshotFormatter, "wpt.fyi screenshots") 395 396 commandline.add_logging_group(parser) 397 return parser 398 399 400def set_from_config(kwargs): 401 if kwargs["config"] is None: 402 config_path = config.path() 403 else: 404 config_path = kwargs["config"] 405 406 kwargs["config_path"] = config_path 407 408 kwargs["config"] = config.read(kwargs["config_path"]) 409 410 keys = {"paths": [("prefs", "prefs_root", True), 411 ("run_info", "run_info", True)], 412 "web-platform-tests": [("remote_url", "remote_url", False), 413 ("branch", "branch", False), 414 ("sync_path", "sync_path", True)], 415 "SSL": [("openssl_binary", "openssl_binary", True), 416 ("certutil_binary", "certutil_binary", True), 417 ("ca_cert_path", "ca_cert_path", True), 418 ("host_cert_path", "host_cert_path", True), 419 ("host_key_path", "host_key_path", True)]} 420 421 for section, values in keys.items(): 422 for config_value, kw_value, is_path in values: 423 if kw_value in kwargs and kwargs[kw_value] is None: 424 if not is_path: 425 new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value) 426 else: 427 new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value) 428 kwargs[kw_value] = new_value 429 430 kwargs["test_paths"] = get_test_paths(kwargs["config"]) 431 432 if kwargs["tests_root"]: 433 if "/" not in kwargs["test_paths"]: 434 kwargs["test_paths"]["/"] = {} 435 kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"] 436 437 if kwargs["metadata_root"]: 438 if "/" not in kwargs["test_paths"]: 439 kwargs["test_paths"]["/"] = {} 440 kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"] 441 442 if kwargs.get("manifest_path"): 443 if "/" not in kwargs["test_paths"]: 444 kwargs["test_paths"]["/"] = {} 445 kwargs["test_paths"]["/"]["manifest_path"] = kwargs["manifest_path"] 446 447 kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests") 448 449 450 check_paths(kwargs) 451 452 453def get_test_paths(config): 454 # Set up test_paths 455 test_paths = OrderedDict() 456 457 for section in config.keys(): 458 if section.startswith("manifest:"): 459 manifest_opts = config.get(section) 460 url_base = manifest_opts.get("url_base", "/") 461 test_paths[url_base] = { 462 "tests_path": manifest_opts.get_path("tests"), 463 "metadata_path": manifest_opts.get_path("metadata"), 464 } 465 if "manifest" in manifest_opts: 466 test_paths[url_base]["manifest_path"] = manifest_opts.get_path("manifest") 467 468 return test_paths 469 470 471def exe_path(name): 472 if name is None: 473 return 474 475 path = find_executable(name) 476 if path and os.access(path, os.X_OK): 477 return path 478 else: 479 return None 480 481 482def check_paths(kwargs): 483 for test_paths in kwargs["test_paths"].values(): 484 if not ("tests_path" in test_paths and 485 "metadata_path" in test_paths): 486 print("Fatal: must specify both a test path and metadata path") 487 sys.exit(1) 488 if "manifest_path" not in test_paths: 489 test_paths["manifest_path"] = os.path.join(test_paths["metadata_path"], 490 "MANIFEST.json") 491 for key, path in test_paths.items(): 492 name = key.split("_", 1)[0] 493 494 if name == "manifest": 495 # For the manifest we can create it later, so just check the path 496 # actually exists 497 path = os.path.dirname(path) 498 499 if not os.path.exists(path): 500 print("Fatal: %s path %s does not exist" % (name, path)) 501 sys.exit(1) 502 503 if not os.path.isdir(path): 504 print("Fatal: %s path %s is not a directory" % (name, path)) 505 sys.exit(1) 506 507 508def check_args(kwargs): 509 set_from_config(kwargs) 510 511 if kwargs["product"] is None: 512 kwargs["product"] = "firefox" 513 514 if kwargs["manifest_update"] is None: 515 kwargs["manifest_update"] = True 516 517 if "sauce" in kwargs["product"]: 518 kwargs["pause_after_test"] = False 519 520 if kwargs["test_list"]: 521 if kwargs["include"] is not None: 522 kwargs["include"].extend(kwargs["test_list"]) 523 else: 524 kwargs["include"] = kwargs["test_list"] 525 526 if kwargs["run_info"] is None: 527 kwargs["run_info"] = kwargs["config_path"] 528 529 if kwargs["this_chunk"] > 1: 530 require_arg(kwargs, "total_chunks", lambda x: x >= kwargs["this_chunk"]) 531 532 if kwargs["chunk_type"] is None: 533 if kwargs["total_chunks"] > 1: 534 kwargs["chunk_type"] = "dir_hash" 535 else: 536 kwargs["chunk_type"] = "none" 537 538 if kwargs["test_groups_file"] is not None: 539 if kwargs["run_by_dir"] is not False: 540 print("Can't pass --test-groups and --run-by-dir") 541 sys.exit(1) 542 if not os.path.exists(kwargs["test_groups_file"]): 543 print("--test-groups file %s not found" % kwargs["test_groups_file"]) 544 sys.exit(1) 545 546 if kwargs["processes"] is None: 547 kwargs["processes"] = 1 548 549 if kwargs["debugger"] is not None: 550 import mozdebug 551 if kwargs["debugger"] == "__default__": 552 kwargs["debugger"] = mozdebug.get_default_debugger_name() 553 debug_info = mozdebug.get_debugger_info(kwargs["debugger"], 554 kwargs["debugger_args"]) 555 if debug_info and debug_info.interactive: 556 if kwargs["processes"] != 1: 557 kwargs["processes"] = 1 558 kwargs["no_capture_stdio"] = True 559 kwargs["debug_info"] = debug_info 560 else: 561 kwargs["debug_info"] = None 562 563 if kwargs["binary"] is not None: 564 if not os.path.exists(kwargs["binary"]): 565 print("Binary path %s does not exist" % kwargs["binary"], file=sys.stderr) 566 sys.exit(1) 567 568 if kwargs["ssl_type"] is None: 569 if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]): 570 kwargs["ssl_type"] = "pregenerated" 571 elif exe_path(kwargs["openssl_binary"]) is not None: 572 kwargs["ssl_type"] = "openssl" 573 else: 574 kwargs["ssl_type"] = "none" 575 576 if kwargs["ssl_type"] == "pregenerated": 577 require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x)) 578 require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x)) 579 require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x)) 580 581 elif kwargs["ssl_type"] == "openssl": 582 path = exe_path(kwargs["openssl_binary"]) 583 if path is None: 584 print("openssl-binary argument missing or not a valid executable", file=sys.stderr) 585 sys.exit(1) 586 kwargs["openssl_binary"] = path 587 588 if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox" and kwargs["certutil_binary"]: 589 path = exe_path(kwargs["certutil_binary"]) 590 if path is None: 591 print("certutil-binary argument missing or not a valid executable", file=sys.stderr) 592 sys.exit(1) 593 kwargs["certutil_binary"] = path 594 595 if kwargs['extra_prefs']: 596 missing = any('=' not in prefarg for prefarg in kwargs['extra_prefs']) 597 if missing: 598 print("Preferences via --setpref must be in key=value format", file=sys.stderr) 599 sys.exit(1) 600 kwargs['extra_prefs'] = [tuple(prefarg.split('=', 1)) for prefarg in 601 kwargs['extra_prefs']] 602 603 if kwargs["reftest_internal"] is None: 604 kwargs["reftest_internal"] = True 605 606 if kwargs["reftest_screenshot"] is None: 607 kwargs["reftest_screenshot"] = "unexpected" if not kwargs["debug_test"] else "always" 608 609 if kwargs["enable_webrender"] is None: 610 kwargs["enable_webrender"] = False 611 612 if kwargs["preload_browser"] is None: 613 # Default to preloading a gecko instance if we're only running a single process 614 kwargs["preload_browser"] = kwargs["processes"] == 1 615 616 return kwargs 617 618 619def check_args_update(kwargs): 620 set_from_config(kwargs) 621 622 if kwargs["product"] is None: 623 kwargs["product"] = "firefox" 624 if kwargs["patch"] is None: 625 kwargs["patch"] = kwargs["sync"] 626 627 for item in kwargs["run_log"]: 628 if os.path.isdir(item): 629 print("Log file %s is a directory" % item, file=sys.stderr) 630 sys.exit(1) 631 632 return kwargs 633 634 635def create_parser_update(product_choices=None): 636 from mozlog.structured import commandline 637 638 from . import products 639 640 if product_choices is None: 641 config_data = config.load() 642 product_choices = products.products_enabled(config_data) 643 644 parser = argparse.ArgumentParser("web-platform-tests-update", 645 description="Update script for web-platform-tests tests.") 646 parser.add_argument("--product", action="store", choices=product_choices, 647 default=None, help="Browser for which metadata is being updated") 648 parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") 649 parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", 650 help="Path to the folder containing test metadata"), 651 parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", 652 help="Path to web-platform-tests"), 653 parser.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path", 654 help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)") 655 parser.add_argument("--sync-path", action="store", type=abs_path, 656 help="Path to store git checkout of web-platform-tests during update"), 657 parser.add_argument("--remote_url", action="store", 658 help="URL of web-platfrom-tests repository to sync against"), 659 parser.add_argument("--branch", action="store", type=abs_path, 660 help="Remote branch to sync against") 661 parser.add_argument("--rev", action="store", help="Revision to sync to") 662 parser.add_argument("--patch", action="store_true", dest="patch", default=None, 663 help="Create a VCS commit containing the changes.") 664 parser.add_argument("--no-patch", action="store_false", dest="patch", 665 help="Don't create a VCS commit containing the changes.") 666 parser.add_argument("--sync", dest="sync", action="store_true", default=False, 667 help="Sync the tests with the latest from upstream (implies --patch)") 668 parser.add_argument("--full", action="store_true", default=False, 669 help=("For all tests that are updated, remove any existing conditions and missing subtests")) 670 parser.add_argument("--disable-intermittent", nargs="?", action="store", const="unstable", default=None, 671 help=("Reason for disabling tests. When updating test results, disable tests that have " 672 "inconsistent results across many runs with the given reason.")) 673 parser.add_argument("--update-intermittent", action="store_true", default=False, 674 help=("Update test metadata with expected intermittent statuses.")) 675 parser.add_argument("--remove-intermittent", action="store_true", default=False, 676 help=("Remove obsolete intermittent statuses from expected statuses.")) 677 parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True, 678 help=("Don't remove metadata files that no longer correspond to a test file")) 679 parser.add_argument("--no-store-state", action="store_false", dest="store_state", 680 help="Store state so that steps can be resumed after failure") 681 parser.add_argument("--continue", action="store_true", 682 help="Continue a previously started run of the update script") 683 parser.add_argument("--abort", action="store_true", 684 help="Clear state from a previous incomplete run of the update script") 685 parser.add_argument("--exclude", action="store", nargs="*", 686 help="List of glob-style paths to exclude when syncing tests") 687 parser.add_argument("--include", action="store", nargs="*", 688 help="List of glob-style paths to include which would otherwise be excluded when syncing tests") 689 parser.add_argument("--extra-property", action="append", default=[], 690 help="Extra property from run_info.json to use in metadata update") 691 # Should make this required iff run=logfile 692 parser.add_argument("run_log", nargs="*", type=abs_path, 693 help="Log file from run of tests") 694 commandline.add_logging_group(parser) 695 return parser 696 697 698def create_parser_reduce(product_choices=None): 699 parser = create_parser(product_choices) 700 parser.add_argument("target", action="store", help="Test id that is unstable") 701 return parser 702 703 704def parse_args(): 705 parser = create_parser() 706 rv = vars(parser.parse_args()) 707 check_args(rv) 708 return rv 709 710 711def parse_args_update(): 712 parser = create_parser_update() 713 rv = vars(parser.parse_args()) 714 check_args_update(rv) 715 return rv 716 717 718def parse_args_reduce(): 719 parser = create_parser_reduce() 720 rv = vars(parser.parse_args()) 721 check_args(rv) 722 return rv 723