1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/ 4 5# ALL CHANGES TO THIS FILE MUST HAVE REVIEW FROM A MARIONETTE PEER! 6# 7# The Marionette Python client is used out-of-tree with various builds of 8# Firefox. Removing a preference from this file will cause regressions, 9# so please be careful and get review from a Testing :: Marionette peer 10# before you make any changes to this file. 11 12from __future__ import absolute_import 13 14import codecs 15import os 16import sys 17import tempfile 18import time 19import traceback 20 21from copy import deepcopy 22 23import mozversion 24 25from mozprofile import Profile 26from mozrunner import Runner, FennecEmulatorRunner 27import six 28from six import reraise 29 30from . import errors 31 32 33class GeckoInstance(object): 34 required_prefs = { 35 # Make sure Shield doesn't hit the network. 36 "app.normandy.api_url": "", 37 38 # Increase the APZ content response timeout in tests to 1 minute. 39 # This is to accommodate the fact that test environments tends to be slower 40 # than production environments (with the b2g emulator being the slowest of them 41 # all), resulting in the production timeout value sometimes being exceeded 42 # and causing false-positive test failures. See bug 1176798, bug 1177018, 43 # bug 1210465. 44 "apz.content_response_timeout": 60000, 45 46 # Do not send Firefox health reports to the production server 47 "datareporting.healthreport.documentServerURI": "http://%(server)s/dummy/healthreport/", 48 49 # Do not show datareporting policy notifications which can interfer with tests 50 "datareporting.policy.dataSubmissionPolicyBypassNotification": True, 51 52 # Automatically unload beforeunload alerts 53 "dom.disable_beforeunload": True, 54 55 # Disable the ProcessHangMonitor 56 "dom.ipc.reportProcessHangs": False, 57 58 # No slow script dialogs 59 "dom.max_chrome_script_run_time": 0, 60 "dom.max_script_run_time": 0, 61 62 # DOM Push 63 "dom.push.connection.enabled": False, 64 65 # Only load extensions from the application and user profile 66 # AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION 67 "extensions.autoDisableScopes": 0, 68 "extensions.enabledScopes": 5, 69 # Disable metadata caching for installed add-ons by default 70 "extensions.getAddons.cache.enabled": False, 71 # Disable intalling any distribution add-ons 72 "extensions.installDistroAddons": False, 73 74 # Turn off extension updates so they don't bother tests 75 "extensions.update.enabled": False, 76 "extensions.update.notifyUser": False, 77 # Make sure opening about:addons won"t hit the network 78 "extensions.getAddons.discovery.api_url": "data:, ", 79 80 # Allow the application to have focus even it runs in the background 81 "focusmanager.testmode": True, 82 83 # Disable useragent updates 84 "general.useragent.updates.enabled": False, 85 86 # Always use network provider for geolocation tests 87 # so we bypass the OSX dialog raised by the corelocation provider 88 "geo.provider.testing": True, 89 # Do not scan Wifi 90 "geo.wifi.scan": False, 91 92 "javascript.options.showInConsole": True, 93 94 # (deprecated and can be removed when Firefox 60 ships) 95 "marionette.defaultPrefs.enabled": True, 96 97 # Disable recommended automation prefs in CI 98 "marionette.prefs.recommended": False, 99 100 # Disable download and usage of OpenH264, and Widevine plugins 101 "media.gmp-manager.updateEnabled": False, 102 103 "media.volume_scale": "0.01", 104 105 # Do not prompt for temporary redirects 106 "network.http.prompt-temp-redirect": False, 107 # Do not automatically switch between offline and online 108 "network.manage-offline-status": False, 109 # Make sure SNTP requests don't hit the network 110 "network.sntp.pools": "%(server)s", 111 112 # Privacy and Tracking Protection 113 "privacy.trackingprotection.enabled": False, 114 115 # Don't do network connections for mitm priming 116 "security.certerrors.mitm.priming.enabled": False, 117 118 # Tests don't wait for the notification button security delay 119 "security.notification_enable_delay": 0, 120 121 # Ensure blocklist updates don't hit the network 122 "services.settings.server": "http://%(server)s/dummy/blocklist/", 123 124 # Disable password capture, so that tests that include forms aren"t 125 # influenced by the presence of the persistent doorhanger notification 126 "signon.rememberSignons": False, 127 128 # Prevent starting into safe mode after application crashes 129 "toolkit.startup.max_resumed_crashes": -1, 130 131 # We want to collect telemetry, but we don't want to send in the results 132 "toolkit.telemetry.server": "https://%(server)s/dummy/telemetry/", 133 134 # Enabling the support for File object creation in the content process. 135 "dom.file.createInChild": True, 136 } 137 138 def __init__(self, host=None, port=None, bin=None, profile=None, addons=None, 139 app_args=None, symbols_path=None, gecko_log=None, prefs=None, 140 workspace=None, verbose=0, headless=False, enable_webrender=False): 141 self.runner_class = Runner 142 self.app_args = app_args or [] 143 self.runner = None 144 self.symbols_path = symbols_path 145 self.binary = bin 146 147 self.marionette_host = host 148 self.marionette_port = port 149 self.addons = addons 150 self.prefs = prefs 151 self.required_prefs = deepcopy(self.required_prefs) 152 if prefs: 153 self.required_prefs.update(prefs) 154 155 self._gecko_log_option = gecko_log 156 self._gecko_log = None 157 self.verbose = verbose 158 self.headless = headless 159 self.enable_webrender = enable_webrender 160 161 # keep track of errors to decide whether instance is unresponsive 162 self.unresponsive_count = 0 163 164 # Alternative to default temporary directory 165 self.workspace = workspace 166 167 # Don't use the 'profile' property here, because sub-classes could add 168 # further preferences and data, which would not be included in the new 169 # profile 170 self._profile = profile 171 172 @property 173 def gecko_log(self): 174 if self._gecko_log: 175 return self._gecko_log 176 177 path = self._gecko_log_option 178 if path != "-": 179 if path is None: 180 path = "gecko.log" 181 elif os.path.isdir(path): 182 fname = "gecko-{}.log".format(time.time()) 183 path = os.path.join(path, fname) 184 185 path = os.path.realpath(path) 186 if os.access(path, os.F_OK): 187 os.remove(path) 188 189 self._gecko_log = path 190 return self._gecko_log 191 192 @property 193 def profile(self): 194 return self._profile 195 196 @profile.setter 197 def profile(self, value): 198 self._update_profile(value) 199 200 def _update_profile(self, profile=None, profile_name=None): 201 """Check if the profile has to be created, or replaced 202 203 :param profile: A Profile instance to be used. 204 :param name: Profile name to be used in the path. 205 """ 206 if self.runner and self.runner.is_running(): 207 raise errors.MarionetteException("The current profile can only be updated " 208 "when the instance is not running") 209 210 if isinstance(profile, Profile): 211 # Only replace the profile if it is not the current one 212 if hasattr(self, "_profile") and profile is self._profile: 213 return 214 215 else: 216 profile_args = self.profile_args 217 profile_path = profile 218 219 # If a path to a profile is given then clone it 220 if isinstance(profile_path, six.string_types): 221 profile_args["path_from"] = profile_path 222 profile_args["path_to"] = tempfile.mkdtemp( 223 suffix=u".{}".format(profile_name or os.path.basename(profile_path)), 224 dir=self.workspace) 225 # The target must not exist yet 226 os.rmdir(profile_args["path_to"]) 227 228 profile = Profile.clone(**profile_args) 229 230 # Otherwise create a new profile 231 else: 232 profile_args["profile"] = tempfile.mkdtemp( 233 suffix=u".{}".format(profile_name or "mozrunner"), 234 dir=self.workspace) 235 profile = Profile(**profile_args) 236 profile.create_new = True 237 238 if isinstance(self.profile, Profile): 239 self.profile.cleanup() 240 241 self._profile = profile 242 243 def switch_profile(self, profile_name=None, clone_from=None): 244 """Switch the profile by using the given name, and optionally clone it. 245 246 Compared to :attr:`profile` this method allows to switch the profile 247 by giving control over the profile name as used for the new profile. It 248 also always creates a new blank profile, or as clone of an existent one. 249 250 :param profile_name: Optional, name of the profile, which will be used 251 as part of the profile path (folder name containing the profile). 252 :clone_from: Optional, if specified the new profile will be cloned 253 based on the given profile. This argument can be an instance of 254 ``mozprofile.Profile``, or the path of the profile. 255 """ 256 if isinstance(clone_from, Profile): 257 clone_from = clone_from.profile 258 259 self._update_profile(clone_from, profile_name=profile_name) 260 261 @property 262 def profile_args(self): 263 args = {"preferences": deepcopy(self.required_prefs)} 264 args["preferences"]["marionette.port"] = self.marionette_port 265 args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port 266 267 if self.prefs: 268 args["preferences"].update(self.prefs) 269 270 if self.verbose: 271 level = "Trace" if self.verbose >= 2 else "Debug" 272 args["preferences"]["marionette.log.level"] = level 273 args["preferences"]["marionette.logging"] = level 274 275 if "-jsdebugger" in self.app_args: 276 args["preferences"].update({ 277 "devtools.browsertoolbox.panel": "jsdebugger", 278 "devtools.debugger.remote-enabled": True, 279 "devtools.chrome.enabled": True, 280 "devtools.debugger.prompt-connection": False, 281 "marionette.debugging.clicktostart": True, 282 }) 283 284 if self.addons: 285 args["addons"] = self.addons 286 287 return args 288 289 @classmethod 290 def create(cls, app=None, *args, **kwargs): 291 try: 292 if not app and kwargs["bin"] is not None: 293 app_id = mozversion.get_version(binary=kwargs["bin"])["application_id"] 294 app = app_ids[app_id] 295 296 instance_class = apps[app] 297 except (IOError, KeyError): 298 exc, val, tb = sys.exc_info() 299 msg = 'Application "{0}" unknown (should be one of {1})'.format(app, apps.keys()) 300 reraise(NotImplementedError, NotImplementedError(msg), tb) 301 302 return instance_class(*args, **kwargs) 303 304 def start(self): 305 self._update_profile(self.profile) 306 self.runner = self.runner_class(**self._get_runner_args()) 307 self.runner.start() 308 309 def _get_runner_args(self): 310 process_args = { 311 "processOutputLine": [NullOutput()], 312 "universal_newlines": True, 313 } 314 315 if self.gecko_log == "-": 316 if six.PY2: 317 process_args["stream"] = codecs.getwriter('utf-8')(sys.stdout) 318 else: 319 process_args["stream"] = codecs.getwriter('utf-8')(sys.stdout.buffer) 320 else: 321 process_args["logfile"] = self.gecko_log 322 323 env = os.environ.copy() 324 325 if self.headless: 326 env["MOZ_HEADLESS"] = "1" 327 env["DISPLAY"] = "77" # Set a fake display. 328 329 if self.enable_webrender: 330 env["MOZ_WEBRENDER"] = "1" 331 env["MOZ_ACCELERATED"] = "1" 332 else: 333 env["MOZ_WEBRENDER"] = "0" 334 335 # environment variables needed for crashreporting 336 # https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting 337 env.update({"MOZ_CRASHREPORTER": "1", 338 "MOZ_CRASHREPORTER_NO_REPORT": "1", 339 "MOZ_CRASHREPORTER_SHUTDOWN": "1", 340 }) 341 342 return { 343 "binary": self.binary, 344 "profile": self.profile, 345 "cmdargs": ["-no-remote", "-marionette"] + self.app_args, 346 "env": env, 347 "symbols_path": self.symbols_path, 348 "process_args": process_args 349 } 350 351 def close(self, clean=False): 352 """ 353 Close the managed Gecko process. 354 355 Depending on self.runner_class, setting `clean` to True may also kill 356 the emulator process in which this instance is running. 357 358 :param clean: If True, also perform runner cleanup. 359 """ 360 if self.runner: 361 self.runner.stop() 362 if clean: 363 self.runner.cleanup() 364 365 if clean: 366 if isinstance(self.profile, Profile): 367 self.profile.cleanup() 368 self.profile = None 369 370 def restart(self, prefs=None, clean=True): 371 """ 372 Close then start the managed Gecko process. 373 374 :param prefs: Dictionary of preference names and values. 375 :param clean: If True, reset the profile before starting. 376 """ 377 if prefs: 378 self.prefs = prefs 379 else: 380 self.prefs = None 381 382 self.close(clean=clean) 383 self.start() 384 385 386class FennecInstance(GeckoInstance): 387 fennec_prefs = { 388 # Enable output for dump() and chrome console API 389 "browser.dom.window.dump.enabled": True, 390 "devtools.console.stdout.chrome": True, 391 392 # Disable safebrowsing components 393 "browser.safebrowsing.blockedURIs.enabled": False, 394 "browser.safebrowsing.downloads.enabled": False, 395 "browser.safebrowsing.passwords.enabled": False, 396 "browser.safebrowsing.malware.enabled": False, 397 "browser.safebrowsing.phishing.enabled": False, 398 399 # Do not restore the last open set of tabs if the browser has crashed 400 "browser.sessionstore.resume_from_crash": False, 401 402 # Disable e10s by default 403 "browser.tabs.remote.autostart": False, 404 405 # Do not allow background tabs to be zombified, otherwise for tests that 406 # open additional tabs, the test harness tab itself might get unloaded 407 "browser.tabs.disableBackgroundZombification": True, 408 } 409 410 def __init__(self, emulator_binary=None, avd_home=None, avd=None, 411 adb_path=None, serial=None, connect_to_running_emulator=False, 412 package_name=None, env=None, *args, **kwargs): 413 required_prefs = deepcopy(FennecInstance.fennec_prefs) 414 required_prefs.update(kwargs.get("prefs", {})) 415 416 super(FennecInstance, self).__init__(*args, **kwargs) 417 self.required_prefs.update(required_prefs) 418 419 self.runner_class = FennecEmulatorRunner 420 # runner args 421 self._package_name = package_name 422 self.emulator_binary = emulator_binary 423 self.avd_home = avd_home 424 self.adb_path = adb_path 425 self.avd = avd 426 self.env = env 427 self.serial = serial 428 self.connect_to_running_emulator = connect_to_running_emulator 429 430 @property 431 def package_name(self): 432 """ 433 Name of app to run on emulator. 434 435 Note that FennecInstance does not use self.binary 436 """ 437 if self._package_name is None: 438 self._package_name = "org.mozilla.fennec" 439 user = os.getenv("USER") 440 if user: 441 self._package_name += "_" + user 442 return self._package_name 443 444 def start(self): 445 self._update_profile(self.profile) 446 self.runner = self.runner_class(**self._get_runner_args()) 447 try: 448 if self.connect_to_running_emulator: 449 self.runner.device.connect() 450 self.runner.start() 451 except Exception: 452 exc_cls, exc, tb = sys.exc_info() 453 reraise(exc_cls, exc_cls( 454 "Error possibly due to runner or device args: {}".format(exc)), tb) 455 456 # forward marionette port 457 self.runner.device.device.forward( 458 local="tcp:{}".format(self.marionette_port), 459 remote="tcp:{}".format(self.marionette_port)) 460 461 def _get_runner_args(self): 462 process_args = { 463 "processOutputLine": [NullOutput()], 464 "universal_newlines": True, 465 } 466 467 env = {} if self.env is None else self.env.copy() 468 if self.enable_webrender: 469 env["MOZ_WEBRENDER"] = "1" 470 else: 471 env["MOZ_WEBRENDER"] = "0" 472 473 runner_args = { 474 "app": self.package_name, 475 "avd_home": self.avd_home, 476 "adb_path": self.adb_path, 477 "binary": self.emulator_binary, 478 "env": env, 479 "profile": self.profile, 480 "cmdargs": ["-marionette"] + self.app_args, 481 "symbols_path": self.symbols_path, 482 "process_args": process_args, 483 "logdir": self.workspace or os.getcwd(), 484 "serial": self.serial, 485 } 486 if self.avd: 487 runner_args["avd"] = self.avd 488 489 return runner_args 490 491 def close(self, clean=False): 492 """ 493 Close the managed Gecko process. 494 495 If `clean` is True and the Fennec instance is running in an 496 emulator managed by mozrunner, this will stop the emulator. 497 498 :param clean: If True, also perform runner cleanup. 499 """ 500 super(FennecInstance, self).close(clean) 501 if clean and self.runner and self.runner.device.connected: 502 try: 503 self.runner.device.device.remove_forwards( 504 "tcp:{}".format(self.marionette_port)) 505 self.unresponsive_count = 0 506 except Exception: 507 self.unresponsive_count += 1 508 traceback.print_exception(*sys.exc_info()) 509 510 511class DesktopInstance(GeckoInstance): 512 desktop_prefs = { 513 # Disable Firefox old build background check 514 "app.update.checkInstallTime": False, 515 516 # Disable automatically upgrading Firefox 517 # 518 # Note: Possible update tests could reset or flip the value to allow 519 # updates to be downloaded and applied. 520 "app.update.disabledForTesting": True, 521 # !!! For backward compatibility up to Firefox 64. Only remove 522 # when this Firefox version is no longer supported by the client !!! 523 "app.update.auto": False, 524 525 # Don't show the content blocking introduction panel 526 # We use a larger number than the default 22 to have some buffer 527 # This can be removed once Firefox 69 and 68 ESR and are no longer supported. 528 "browser.contentblocking.introCount": 99, 529 530 # Enable output for dump() and chrome console API 531 "browser.dom.window.dump.enabled": True, 532 "devtools.console.stdout.chrome": True, 533 534 # Indicate that the download panel has been shown once so that whichever 535 # download test runs first doesn"t show the popup inconsistently 536 "browser.download.panel.shown": True, 537 538 # Do not show the EULA notification which can interfer with tests 539 "browser.EULA.override": True, 540 541 # Always display a blank page 542 "browser.newtabpage.enabled": False, 543 544 # Background thumbnails in particular cause grief, and disabling thumbnails 545 # in general can"t hurt - we re-enable them when tests need them 546 "browser.pagethumbnails.capturing_disabled": True, 547 548 # Disable safebrowsing components 549 "browser.safebrowsing.blockedURIs.enabled": False, 550 "browser.safebrowsing.downloads.enabled": False, 551 "browser.safebrowsing.passwords.enabled": False, 552 "browser.safebrowsing.malware.enabled": False, 553 "browser.safebrowsing.phishing.enabled": False, 554 555 # Disable updates to search engines 556 "browser.search.update": False, 557 558 # Do not restore the last open set of tabs if the browser has crashed 559 "browser.sessionstore.resume_from_crash": False, 560 561 # Don't check for the default web browser during startup 562 "browser.shell.checkDefaultBrowser": False, 563 564 # Needed for branded builds to prevent opening a second tab on startup 565 "browser.startup.homepage_override.mstone": "ignore", 566 # Start with a blank page by default 567 "browser.startup.page": 0, 568 569 # Disable browser animations 570 "toolkit.cosmeticAnimations.enabled": False, 571 572 # Bug 1557457: Disable because modal dialogs might not appear in Firefox 573 "browser.tabs.remote.separatePrivilegedContentProcess": False, 574 575 # Don't unload tabs when available memory is running low 576 "browser.tabs.unloadOnLowMemory": False, 577 578 # Do not warn when closing all open tabs 579 "browser.tabs.warnOnClose": False, 580 # Do not warn when closing all other open tabs 581 "browser.tabs.warnOnCloseOtherTabs": False, 582 # Do not warn when multiple tabs will be opened 583 "browser.tabs.warnOnOpen": False, 584 585 # Disable the UI tour 586 "browser.uitour.enabled": False, 587 588 # Turn off search suggestions in the location bar so as not to trigger network 589 # connections. 590 "browser.urlbar.suggest.searches": False, 591 592 # Don't warn when exiting the browser 593 "browser.warnOnQuit": False, 594 595 # Disable first-run welcome page 596 "startup.homepage_welcome_url": "about:blank", 597 "startup.homepage_welcome_url.additional": "", 598 } 599 600 def __init__(self, *args, **kwargs): 601 required_prefs = deepcopy(DesktopInstance.desktop_prefs) 602 required_prefs.update(kwargs.get("prefs", {})) 603 604 super(DesktopInstance, self).__init__(*args, **kwargs) 605 self.required_prefs.update(required_prefs) 606 607 608class ThunderbirdInstance(GeckoInstance): 609 def __init__(self, *args, **kwargs): 610 super(ThunderbirdInstance, self).__init__(*args, **kwargs) 611 try: 612 # Copied alongside in the test archive 613 from .thunderbirdinstance import thunderbird_prefs 614 except ImportError: 615 try: 616 # Coming from source tree through virtualenv 617 from thunderbirdinstance import thunderbird_prefs 618 except ImportError: 619 thunderbird_prefs = {} 620 self.required_prefs.update(thunderbird_prefs) 621 622 623class NullOutput(object): 624 def __call__(self, line): 625 pass 626 627 628apps = { 629 'fennec': FennecInstance, 630 'fxdesktop': DesktopInstance, 631 'thunderbird': ThunderbirdInstance, 632} 633 634app_ids = { 635 '{aa3c5121-dab2-40e2-81ca-7ea25febc110}': 'fennec', 636 '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}': 'fxdesktop', 637 '{3550f703-e582-4d05-9a08-453d09bdfdc6}': 'thunderbird', 638} 639