1import os 2import platform 3import re 4import shutil 5import stat 6import errno 7import subprocess 8import tempfile 9from abc import ABCMeta, abstractmethod 10from datetime import datetime, timedelta 11from distutils.spawn import find_executable 12 13from six.moves.urllib.parse import urlsplit 14import requests 15 16from .utils import call, get, untar, unzip 17 18uname = platform.uname() 19 20# the rootUrl for the firefox-ci deployment of Taskcluster 21# (after November 9, https://firefox-ci-tc.services.mozilla.com/) 22FIREFOX_CI_ROOT_URL = 'https://taskcluster.net' 23 24 25def _get_fileversion(binary, logger=None): 26 command = "(Get-Item '%s').VersionInfo.FileVersion" % binary.replace("'", "''") 27 try: 28 return call("powershell.exe", command).strip() 29 except (subprocess.CalledProcessError, OSError): 30 if logger is not None: 31 logger.warning("Failed to call %s in PowerShell" % command) 32 return None 33 34 35def handle_remove_readonly(func, path, exc): 36 excvalue = exc[1] 37 if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: 38 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 39 func(path) 40 else: 41 raise 42 43 44def get_ext(filename): 45 """Get the extension from a filename with special handling for .tar.foo""" 46 name, ext = os.path.splitext(filename) 47 if name.endswith(".tar"): 48 ext = ".tar%s" % ext 49 return ext 50 51 52class Browser(object): 53 __metaclass__ = ABCMeta 54 55 def __init__(self, logger): 56 self.logger = logger 57 58 @abstractmethod 59 def download(self, dest=None, channel=None, rename=None): 60 """Download a package or installer for the browser 61 :param dest: Directory in which to put the dowloaded package 62 :param channel: Browser channel to download 63 :param rename: Optional name for the downloaded package; the original 64 extension is preserved. 65 """ 66 return NotImplemented 67 68 @abstractmethod 69 def install(self, dest=None): 70 """Install the browser.""" 71 return NotImplemented 72 73 @abstractmethod 74 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 75 """Install the WebDriver implementation for this browser.""" 76 return NotImplemented 77 78 @abstractmethod 79 def find_binary(self, venv_path=None, channel=None): 80 """Find the binary of the browser. 81 82 If the WebDriver for the browser is able to find the binary itself, this 83 method doesn't need to be implemented, in which case NotImplementedError 84 is suggested to be raised to prevent accidental use. 85 """ 86 return NotImplemented 87 88 @abstractmethod 89 def find_webdriver(self, channel=None): 90 """Find the binary of the WebDriver.""" 91 return NotImplemented 92 93 @abstractmethod 94 def version(self, binary=None, webdriver_binary=None): 95 """Retrieve the release version of the installed browser.""" 96 return NotImplemented 97 98 @abstractmethod 99 def requirements(self): 100 """Name of the browser-specific wptrunner requirements file""" 101 return NotImplemented 102 103 104class Firefox(Browser): 105 """Firefox-specific interface. 106 107 Includes installation, webdriver installation, and wptrunner setup methods. 108 """ 109 110 product = "firefox" 111 binary = "browsers/firefox/firefox" 112 requirements = "requirements_firefox.txt" 113 114 platform = { 115 "Linux": "linux", 116 "Windows": "win", 117 "Darwin": "macos" 118 }.get(uname[0]) 119 120 application_name = { 121 "stable": "Firefox.app", 122 "beta": "Firefox.app", 123 "nightly": "Firefox Nightly.app" 124 } 125 126 def platform_string_geckodriver(self): 127 if self.platform is None: 128 raise ValueError("Unable to construct a valid Geckodriver package name for current platform") 129 130 if self.platform in ("linux", "win"): 131 bits = "64" if uname[4] == "x86_64" else "32" 132 else: 133 bits = "" 134 135 return "%s%s" % (self.platform, bits) 136 137 def _get_dest(self, dest, channel): 138 if dest is None: 139 # os.getcwd() doesn't include the venv path 140 dest = os.path.join(os.getcwd(), "_venv") 141 142 dest = os.path.join(dest, "browsers", channel) 143 144 if not os.path.exists(dest): 145 os.makedirs(dest) 146 147 return dest 148 149 def download(self, dest=None, channel="nightly", rename=None): 150 product = { 151 "nightly": "firefox-nightly-latest-ssl", 152 "beta": "firefox-beta-latest-ssl", 153 "stable": "firefox-latest-ssl" 154 } 155 156 os_builds = { 157 ("linux", "x86"): "linux", 158 ("linux", "x86_64"): "linux64", 159 ("win", "x86"): "win", 160 ("win", "AMD64"): "win64", 161 ("macos", "x86_64"): "osx", 162 } 163 os_key = (self.platform, uname[4]) 164 165 if dest is None: 166 dest = self._get_dest(None, channel) 167 168 if channel not in product: 169 raise ValueError("Unrecognised release channel: %s" % channel) 170 171 if os_key not in os_builds: 172 raise ValueError("Unsupported platform: %s %s" % os_key) 173 174 url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel], 175 os_builds[os_key]) 176 self.logger.info("Downloading Firefox from %s" % url) 177 resp = requests.get(url) 178 179 filename = None 180 181 content_disposition = resp.headers.get('content-disposition') 182 if content_disposition: 183 filenames = re.findall("filename=(.+)", content_disposition) 184 if filenames: 185 filename = filenames[0] 186 187 if not filename: 188 filename = urlsplit(resp.url).path.rsplit("/", 1)[1] 189 190 if not filename: 191 filename = "firefox.tar.bz2" 192 193 if rename: 194 filename = "%s%s" % (rename, get_ext(filename)) 195 196 installer_path = os.path.join(dest, filename) 197 198 with open(installer_path, "wb") as f: 199 f.write(resp.content) 200 201 return installer_path 202 203 def install(self, dest=None, channel="nightly"): 204 """Install Firefox.""" 205 import mozinstall 206 207 dest = self._get_dest(dest, channel) 208 209 filename = os.path.basename(dest) 210 211 installer_path = self.download(dest, channel) 212 213 try: 214 mozinstall.install(installer_path, dest) 215 except mozinstall.mozinstall.InstallError: 216 if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))): 217 # mozinstall will fail if nightly is already installed in the venv because 218 # mac installation uses shutil.copy_tree 219 mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))) 220 mozinstall.install(filename, dest) 221 else: 222 raise 223 224 os.remove(installer_path) 225 return self.find_binary_path(dest) 226 227 def find_binary_path(self, path=None, channel="nightly"): 228 """Looks for the firefox binary in the virtual environment""" 229 230 if path is None: 231 # os.getcwd() doesn't include the venv path 232 path = os.path.join(os.getcwd(), "_venv", "browsers", channel) 233 234 binary = None 235 236 if self.platform == "linux": 237 binary = find_executable("firefox", os.path.join(path, "firefox")) 238 elif self.platform == "win": 239 import mozinstall 240 try: 241 binary = mozinstall.get_binary(path, "firefox") 242 except mozinstall.InvalidBinary: 243 # ignore the case where we fail to get a binary 244 pass 245 elif self.platform == "macos": 246 binary = find_executable("firefox", os.path.join(path, self.application_name.get(channel, "Firefox Nightly.app"), 247 "Contents", "MacOS")) 248 249 return binary 250 251 def find_binary(self, venv_path=None, channel="nightly"): 252 if venv_path is None: 253 venv_path = os.path.join(os.getcwd(), "_venv") 254 255 path = os.path.join(venv_path, "browsers", channel) 256 binary = self.find_binary_path(path, channel) 257 258 if not binary and self.platform == "win": 259 winpaths = [os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Mozilla Firefox"), 260 os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Mozilla Firefox")] 261 for winpath in winpaths: 262 binary = self.find_binary_path(winpath, channel) 263 if binary is not None: 264 break 265 266 if not binary and self.platform == "macos": 267 macpaths = ["/Applications/Firefox Nightly.app/Contents/MacOS", 268 os.path.expanduser("~/Applications/Firefox Nightly.app/Contents/MacOS"), 269 "/Applications/Firefox Developer Edition.app/Contents/MacOS", 270 os.path.expanduser("~/Applications/Firefox Developer Edition.app/Contents/MacOS"), 271 "/Applications/Firefox.app/Contents/MacOS", 272 os.path.expanduser("~/Applications/Firefox.app/Contents/MacOS")] 273 return find_executable("firefox", os.pathsep.join(macpaths)) 274 275 if binary is None: 276 return find_executable("firefox") 277 278 return binary 279 280 def find_certutil(self): 281 path = find_executable("certutil") 282 if path is None: 283 return None 284 if os.path.splitdrive(os.path.normcase(path))[1].split(os.path.sep) == ["", "windows", "system32", "certutil.exe"]: 285 return None 286 return path 287 288 def find_webdriver(self, channel=None): 289 return find_executable("geckodriver") 290 291 def get_version_and_channel(self, binary): 292 version_string = call(binary, "--version").strip() 293 m = re.match(r"Mozilla Firefox (\d+\.\d+(?:\.\d+)?)(a|b)?", version_string) 294 if not m: 295 return None, "nightly" 296 version, status = m.groups() 297 channel = {"a": "nightly", "b": "beta"} 298 return version, channel.get(status, "stable") 299 300 def get_profile_bundle_url(self, version, channel): 301 if channel == "stable": 302 repo = "https://hg.mozilla.org/releases/mozilla-release" 303 tag = "FIREFOX_%s_RELEASE" % version.replace(".", "_") 304 elif channel == "beta": 305 repo = "https://hg.mozilla.org/releases/mozilla-beta" 306 major_version = version.split(".", 1)[0] 307 # For beta we have a different format for betas that are now in stable releases 308 # vs those that are not 309 tags = get("https://hg.mozilla.org/releases/mozilla-beta/json-tags").json()["tags"] 310 tags = {item["tag"] for item in tags} 311 end_tag = "FIREFOX_BETA_%s_END" % major_version 312 if end_tag in tags: 313 tag = end_tag 314 else: 315 tag = "tip" 316 else: 317 repo = "https://hg.mozilla.org/mozilla-central" 318 # Always use tip as the tag for nightly; this isn't quite right 319 # but to do better we need the actual build revision, which we 320 # can get if we have an application.ini file 321 tag = "tip" 322 323 return "%s/archive/%s.zip/testing/profiles/" % (repo, tag) 324 325 def install_prefs(self, binary, dest=None, channel=None): 326 if binary: 327 version, channel_ = self.get_version_and_channel(binary) 328 if channel is not None and channel != channel_: 329 # Beta doesn't always seem to have the b in the version string, so allow the 330 # manually supplied value to override the one from the binary 331 self.logger.warning("Supplied channel doesn't match binary, using supplied channel") 332 elif channel is None: 333 channel = channel_ 334 else: 335 version = None 336 337 if dest is None: 338 dest = os.curdir 339 340 dest = os.path.join(dest, "profiles", channel) 341 if version: 342 dest = os.path.join(dest, version) 343 have_cache = False 344 if os.path.exists(dest) and len(os.listdir(dest)) > 0: 345 if channel != "nightly": 346 have_cache = True 347 else: 348 now = datetime.now() 349 have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) > 350 now - timedelta(days=1)) 351 352 # If we don't have a recent download, grab and extract the latest one 353 if not have_cache: 354 if os.path.exists(dest): 355 shutil.rmtree(dest) 356 os.makedirs(dest) 357 358 url = self.get_profile_bundle_url(version, channel) 359 360 self.logger.info("Installing test prefs from %s" % url) 361 try: 362 extract_dir = tempfile.mkdtemp() 363 unzip(get(url).raw, dest=extract_dir) 364 365 profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles') 366 for name in os.listdir(profiles): 367 path = os.path.join(profiles, name) 368 shutil.move(path, dest) 369 finally: 370 shutil.rmtree(extract_dir) 371 else: 372 self.logger.info("Using cached test prefs from %s" % dest) 373 374 return dest 375 376 def _latest_geckodriver_version(self): 377 """Get and return latest version number for geckodriver.""" 378 # This is used rather than an API call to avoid rate limits 379 tags = call("git", "ls-remote", "--tags", "--refs", 380 "https://github.com/mozilla/geckodriver.git") 381 release_re = re.compile(r".*refs/tags/v(\d+)\.(\d+)\.(\d+)") 382 latest_release = 0 383 for item in tags.split("\n"): 384 m = release_re.match(item) 385 if m: 386 version = [int(item) for item in m.groups()] 387 if version > latest_release: 388 latest_release = version 389 assert latest_release != 0 390 return "v%s.%s.%s" % tuple(str(item) for item in latest_release) 391 392 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 393 """Install latest Geckodriver.""" 394 if dest is None: 395 dest = os.getcwd() 396 397 if channel == "nightly": 398 path = self.install_geckodriver_nightly(dest) 399 if path is not None: 400 return path 401 else: 402 self.logger.warning("Nightly webdriver not found; falling back to release") 403 404 version = self._latest_geckodriver_version() 405 format = "zip" if uname[0] == "Windows" else "tar.gz" 406 self.logger.debug("Latest geckodriver release %s" % version) 407 url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" % 408 (version, version, self.platform_string_geckodriver(), format)) 409 if format == "zip": 410 unzip(get(url).raw, dest=dest) 411 else: 412 untar(get(url).raw, dest=dest) 413 return find_executable(os.path.join(dest, "geckodriver")) 414 415 def install_geckodriver_nightly(self, dest): 416 import tarfile 417 import mozdownload 418 self.logger.info("Attempting to install webdriver from nightly") 419 try: 420 s = mozdownload.DailyScraper(branch="mozilla-central", 421 extension="common.tests.tar.gz", 422 destination=dest) 423 package_path = s.download() 424 except mozdownload.errors.NotFoundError: 425 return 426 427 try: 428 exe_suffix = ".exe" if uname[0] == "Windows" else "" 429 with tarfile.open(package_path, "r") as f: 430 try: 431 member = f.getmember("bin%sgeckodriver%s" % (os.path.sep, 432 exe_suffix)) 433 except KeyError: 434 return 435 # Remove bin/ from the path. 436 member.name = os.path.basename(member.name) 437 f.extractall(members=[member], path=dest) 438 path = os.path.join(dest, member.name) 439 self.logger.info("Extracted geckodriver to %s" % path) 440 finally: 441 os.unlink(package_path) 442 443 return path 444 445 def version(self, binary=None, webdriver_binary=None): 446 """Retrieve the release version of the installed browser.""" 447 version_string = call(binary, "--version").strip() 448 m = re.match(r"Mozilla Firefox (.*)", version_string) 449 if not m: 450 return None 451 return m.group(1) 452 453 454class FirefoxAndroid(Browser): 455 """Android-specific Firefox interface.""" 456 457 product = "firefox_android" 458 requirements = "requirements_firefox.txt" 459 460 def download(self, dest=None, channel=None, rename=None): 461 if dest is None: 462 dest = os.pwd 463 464 if FIREFOX_CI_ROOT_URL == 'https://taskcluster.net': 465 # NOTE: this condition can be removed after November 9, 2019 466 TC_QUEUE_BASE = "https://queue.taskcluster.net/v1/" 467 TC_INDEX_BASE = "https://index.taskcluster.net/v1/" 468 else: 469 TC_QUEUE_BASE = FIREFOX_CI_ROOT_URL + "/api/queue/v1/" 470 TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/" 471 472 473 resp = requests.get(TC_INDEX_BASE + 474 "task/gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt") 475 resp.raise_for_status() 476 index = resp.json() 477 task_id = index["taskId"] 478 479 resp = requests.get(TC_QUEUE_BASE + "task/%s/artifacts/%s" % 480 (task_id, "public/build/geckoview-androidTest.apk")) 481 resp.raise_for_status() 482 483 filename = "geckoview-androidTest.apk" 484 if rename: 485 filename = "%s%s" % (rename, get_ext(filename)[1]) 486 apk_path = os.path.join(dest, filename) 487 488 with open(apk_path, "wb") as f: 489 f.write(resp.content) 490 491 return apk_path 492 493 def install(self, dest=None, channel=None): 494 return self.download(dest, channel) 495 496 def install_prefs(self, binary, dest=None, channel=None): 497 fx_browser = Firefox(self.logger) 498 return fx_browser.install_prefs(binary, dest, channel) 499 500 def find_binary(self, venv_path=None, channel=None): 501 raise NotImplementedError 502 503 def find_webdriver(self, channel=None): 504 raise NotImplementedError 505 506 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 507 raise NotImplementedError 508 509 def version(self, binary=None, webdriver_binary=None): 510 return None 511 512 513class Chrome(Browser): 514 """Chrome-specific interface. 515 516 Includes webdriver installation, and wptrunner setup methods. 517 """ 518 519 product = "chrome" 520 requirements = "requirements_chrome.txt" 521 522 def download(self, dest=None, channel=None, rename=None): 523 raise NotImplementedError 524 525 def install(self, dest=None, channel=None): 526 raise NotImplementedError 527 528 def platform_string(self): 529 platform = { 530 "Linux": "linux", 531 "Windows": "win", 532 "Darwin": "mac" 533 }.get(uname[0]) 534 535 if platform is None: 536 raise ValueError("Unable to construct a valid Chrome package name for current platform") 537 538 if platform == "linux": 539 bits = "64" if uname[4] == "x86_64" else "32" 540 elif platform == "mac": 541 bits = "64" 542 elif platform == "win": 543 bits = "32" 544 545 return "%s%s" % (platform, bits) 546 547 def chromium_platform_string(self): 548 platform = { 549 "Linux": "Linux", 550 "Windows": "Win", 551 "Darwin": "Mac" 552 }.get(uname[0]) 553 554 if platform is None: 555 raise ValueError("Unable to construct a valid Chromium package name for current platform") 556 557 if (platform == "Linux" or platform == "Win") and uname[4] == "x86_64": 558 platform += "_x64" 559 560 return platform 561 562 def find_binary(self, venv_path=None, channel=None): 563 if uname[0] == "Linux": 564 name = "google-chrome" 565 if channel == "stable": 566 name += "-stable" 567 elif channel == "beta": 568 name += "-beta" 569 elif channel == "dev": 570 name += "-unstable" 571 # No Canary on Linux. 572 return find_executable(name) 573 if uname[0] == "Darwin": 574 if channel == "canary": 575 return "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary" 576 # All other channels share the same path on macOS. 577 return "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" 578 if uname[0] == "Windows": 579 return os.path.expandvars(r"$SYSTEMDRIVE\Program Files (x86)\Google\Chrome\Application\chrome.exe") 580 self.logger.warning("Unable to find the browser binary.") 581 return None 582 583 def find_webdriver(self, channel=None): 584 return find_executable("chromedriver") 585 586 def _official_chromedriver_url(self, chrome_version): 587 # http://chromedriver.chromium.org/downloads/version-selection 588 parts = chrome_version.split(".") 589 assert len(parts) == 4 590 latest_url = "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_%s.%s.%s" % tuple(parts[:-1]) 591 try: 592 latest = get(latest_url).text.strip() 593 except requests.RequestException: 594 latest_url = "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_%s" % parts[0] 595 try: 596 latest = get(latest_url).text.strip() 597 except requests.RequestException: 598 return None 599 return "https://chromedriver.storage.googleapis.com/%s/chromedriver_%s.zip" % ( 600 latest, self.platform_string()) 601 602 def _chromium_chromedriver_url(self, chrome_version): 603 try: 604 # Try to find the Chromium build with the same revision. 605 omaha = get("https://omahaproxy.appspot.com/deps.json?version=" + chrome_version).json() 606 revision = omaha['chromium_base_position'] 607 url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/%s/chromedriver_%s.zip" % ( 608 self.chromium_platform_string(), revision, self.platform_string()) 609 # Check the status without downloading the content (this is a streaming request). 610 get(url) 611 except requests.RequestException: 612 # Fall back to the tip-of-tree Chromium build. 613 revision_url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/LAST_CHANGE" % ( 614 self.chromium_platform_string()) 615 revision = get(revision_url).text.strip() 616 url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/%s/chromedriver_%s.zip" % ( 617 self.chromium_platform_string(), revision, self.platform_string()) 618 return url 619 620 def _latest_chromedriver_url(self, chrome_version): 621 # Remove channel suffixes (e.g. " dev"). 622 chrome_version = chrome_version.split(' ')[0] 623 return (self._official_chromedriver_url(chrome_version) or 624 self._chromium_chromedriver_url(chrome_version)) 625 626 def install_webdriver_by_version(self, version, dest=None): 627 assert version, "Cannot install ChromeDriver without Chrome version" 628 if dest is None: 629 dest = os.pwd 630 url = self._latest_chromedriver_url(version) 631 self.logger.info("Downloading ChromeDriver from %s" % url) 632 unzip(get(url).raw, dest) 633 chromedriver_dir = os.path.join( 634 dest, 'chromedriver_%s' % self.platform_string()) 635 if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")): 636 shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest) 637 shutil.rmtree(chromedriver_dir) 638 return find_executable("chromedriver", dest) 639 640 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 641 if browser_binary is None: 642 browser_binary = self.find_binary(channel) 643 return self.install_webdriver_by_version( 644 self.version(browser_binary), dest) 645 646 def version(self, binary=None, webdriver_binary=None): 647 if not binary: 648 self.logger.warning("No browser binary provided.") 649 return None 650 651 if uname[0] == "Windows": 652 return _get_fileversion(binary, self.logger) 653 654 try: 655 version_string = call(binary, "--version").strip() 656 except subprocess.CalledProcessError: 657 self.logger.warning("Failed to call %s" % binary) 658 return None 659 m = re.match(r"(?:Google Chrome|Chromium) (.*)", version_string) 660 if not m: 661 self.logger.warning("Failed to extract version from: %s" % version_string) 662 return None 663 return m.group(1) 664 665 666class ChromeAndroidBase(Browser): 667 """A base class for ChromeAndroid and AndroidWebView. 668 669 On Android, WebView is based on Chromium open source project, and on some 670 versions of Android we share the library with Chrome. Therefore, we have 671 a very similar WPT runner implementation. 672 Includes webdriver installation. 673 """ 674 __metaclass__ = ABCMeta # This is an abstract class. 675 676 def __init__(self, logger): 677 super(ChromeAndroidBase, self).__init__(logger) 678 self.device_serial = None 679 680 def download(self, dest=None, channel=None, rename=None): 681 raise NotImplementedError 682 683 def install(self, dest=None, channel=None): 684 raise NotImplementedError 685 686 @abstractmethod 687 def find_binary(self, venv_path=None, channel=None): 688 raise NotImplementedError 689 690 def find_webdriver(self, channel=None): 691 return find_executable("chromedriver") 692 693 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 694 if browser_binary is None: 695 browser_binary = self.find_binary(channel) 696 chrome = Chrome(self.logger) 697 return chrome.install_webdriver_by_version( 698 self.version(browser_binary), dest) 699 700 def version(self, binary=None, webdriver_binary=None): 701 if not binary: 702 self.logger.warning("No package name provided.") 703 return None 704 705 command = ['adb'] 706 if self.device_serial: 707 command.extend(['-s', self.device_serial]) 708 command.extend(['shell', 'dumpsys', 'package', binary]) 709 try: 710 output = call(*command) 711 except (subprocess.CalledProcessError, OSError): 712 self.logger.warning("Failed to call %s" % " ".join(command)) 713 return None 714 match = re.search(r'versionName=(.*)', output) 715 if not match: 716 self.logger.warning("Failed to find versionName") 717 return None 718 return match.group(1) 719 720 721class ChromeAndroid(ChromeAndroidBase): 722 """Chrome-specific interface for Android. 723 """ 724 725 product = "chrome_android" 726 requirements = "requirements_chrome_android.txt" 727 728 def find_binary(self, venv_path=None, channel=None): 729 if channel in ("beta", "dev", "canary"): 730 return "com.chrome." + channel 731 return "com.android.chrome" 732 733 734#TODO(aluo): This is largely copied from the AndroidWebView implementation. 735# Tests are not running for weblayer yet (crbug/1019521), this 736# initial implementation will help to reproduce and debug any issues. 737class AndroidWeblayer(ChromeAndroidBase): 738 """Weblayer-specific interface for Android.""" 739 740 product = "android_weblayer" 741 # TODO(aluo): replace this with weblayer version after tests are working. 742 requirements = "requirements_android_webview.txt" 743 744 def find_binary(self, venv_path=None, channel=None): 745 return "org.chromium.weblayer.shell" 746 747 748class AndroidWebview(ChromeAndroidBase): 749 """Webview-specific interface for Android. 750 751 Design doc: 752 https://docs.google.com/document/d/19cGz31lzCBdpbtSC92svXlhlhn68hrsVwSB7cfZt54o/view 753 """ 754 755 product = "android_webview" 756 requirements = "requirements_android_webview.txt" 757 758 def find_binary(self, venv_path=None, channel=None): 759 # Just get the current package name of the WebView provider. 760 # For WebView, it is not trivial to change the WebView provider, so 761 # we will just grab whatever is available. 762 # https://chromium.googlesource.com/chromium/src/+/HEAD/android_webview/docs/channels.md 763 command = ['adb'] 764 if self.device_serial: 765 command.extend(['-s', self.device_serial]) 766 command.extend(['shell', 'dumpsys', 'webviewupdate']) 767 try: 768 output = call(*command) 769 except (subprocess.CalledProcessError, OSError): 770 self.logger.warning("Failed to call %s" % " ".join(command)) 771 return None 772 m = re.search(r'^\s*Current WebView package \(name, version\): \((.*), ([0-9.]*)\)$', 773 output, re.M) 774 if m is None: 775 self.logger.warning("Unable to find current WebView package in dumpsys output") 776 return None 777 self.logger.warning("Final package name: " + m.group(1)) 778 return m.group(1) 779 780 781class ChromeiOS(Browser): 782 """Chrome-specific interface for iOS. 783 """ 784 785 product = "chrome_ios" 786 requirements = "requirements_chrome_ios.txt" 787 788 def download(self, dest=None, channel=None, rename=None): 789 raise NotImplementedError 790 791 def install(self, dest=None, channel=None): 792 raise NotImplementedError 793 794 def find_binary(self, venv_path=None, channel=None): 795 raise NotImplementedError 796 797 def find_webdriver(self, channel=None): 798 raise NotImplementedError 799 800 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 801 raise NotImplementedError 802 803 def version(self, binary=None, webdriver_binary=None): 804 return None 805 806 807class Opera(Browser): 808 """Opera-specific interface. 809 810 Includes webdriver installation, and wptrunner setup methods. 811 """ 812 813 product = "opera" 814 requirements = "requirements_opera.txt" 815 816 @property 817 def binary(self): 818 if uname[0] == "Linux": 819 return "/usr/bin/opera" 820 # TODO Windows, Mac? 821 self.logger.warning("Unable to find the browser binary.") 822 return None 823 824 def download(self, dest=None, channel=None, rename=None): 825 raise NotImplementedError 826 827 def install(self, dest=None, channel=None): 828 raise NotImplementedError 829 830 def platform_string(self): 831 platform = { 832 "Linux": "linux", 833 "Windows": "win", 834 "Darwin": "mac" 835 }.get(uname[0]) 836 837 if platform is None: 838 raise ValueError("Unable to construct a valid Opera package name for current platform") 839 840 if platform == "linux": 841 bits = "64" if uname[4] == "x86_64" else "32" 842 elif platform == "mac": 843 bits = "64" 844 elif platform == "win": 845 bits = "32" 846 847 return "%s%s" % (platform, bits) 848 849 def find_binary(self, venv_path=None, channel=None): 850 raise NotImplementedError 851 852 def find_webdriver(self, channel=None): 853 return find_executable("operadriver") 854 855 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 856 if dest is None: 857 dest = os.pwd 858 latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"] 859 url = "https://github.com/operasoftware/operachromiumdriver/releases/download/%s/operadriver_%s.zip" % (latest, 860 self.platform_string()) 861 unzip(get(url).raw, dest) 862 863 operadriver_dir = os.path.join(dest, "operadriver_%s" % self.platform_string()) 864 shutil.move(os.path.join(operadriver_dir, "operadriver"), dest) 865 shutil.rmtree(operadriver_dir) 866 867 path = find_executable("operadriver") 868 st = os.stat(path) 869 os.chmod(path, st.st_mode | stat.S_IEXEC) 870 return path 871 872 def version(self, binary=None, webdriver_binary=None): 873 """Retrieve the release version of the installed browser.""" 874 binary = binary or self.binary 875 try: 876 output = call(binary, "--version") 877 except subprocess.CalledProcessError: 878 self.logger.warning("Failed to call %s" % binary) 879 return None 880 m = re.search(r"[0-9\.]+( [a-z]+)?$", output.strip()) 881 if m: 882 return m.group(0) 883 884 885class EdgeChromium(Browser): 886 """MicrosoftEdge-specific interface.""" 887 platform = { 888 "Linux": "linux", 889 "Windows": "win", 890 "Darwin": "macos" 891 }.get(uname[0]) 892 product = "edgechromium" 893 edgedriver_name = "msedgedriver" 894 requirements = "requirements_edge_chromium.txt" 895 896 def download(self, dest=None, channel=None, rename=None): 897 raise NotImplementedError 898 899 def install(self, dest=None, channel=None): 900 raise NotImplementedError 901 902 def find_binary(self, venv_path=None, channel=None): 903 binary = None 904 if self.platform == "win": 905 binaryname = "msedge" 906 binary = find_executable(binaryname) 907 if not binary: 908 # Use paths from different Edge channels starting with Release\Beta\Dev\Canary 909 winpaths = [os.path.expanduser("~\\AppData\\Local\\Microsoft\\Edge\\Application"), 910 os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge\\Application"), 911 os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge Beta\\Application"), 912 os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge Dev\\Application"), 913 os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge\\Application"), 914 os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge Beta\\Application"), 915 os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge Dev\\Application"), 916 os.path.expanduser("~\\AppData\Local\\Microsoft\\Edge SxS\\Application")] 917 return find_executable(binaryname, os.pathsep.join(winpaths)) 918 if self.platform == "macos": 919 binaryname = "Microsoft Edge Canary" 920 binary = find_executable(binaryname) 921 if not binary: 922 macpaths = ["/Applications/Microsoft Edge.app/Contents/MacOS", 923 os.path.expanduser("~/Applications/Microsoft Edge.app/Contents/MacOS"), 924 "/Applications/Microsoft Edge Canary.app/Contents/MacOS", 925 os.path.expanduser("~/Applications/Microsoft Edge Canary.app/Contents/MacOS")] 926 return find_executable("Microsoft Edge Canary", os.pathsep.join(macpaths)) 927 return binary 928 929 def find_webdriver(self, channel=None): 930 return find_executable("msedgedriver") 931 932 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 933 if self.platform != "win" and self.platform != "macos": 934 raise ValueError("Only Windows and Mac platforms are currently supported") 935 936 if dest is None: 937 dest = os.pwd 938 939 if channel is None: 940 version_url = "https://msedgedriver.azureedge.net/LATEST_DEV" 941 else: 942 version_url = "https://msedgedriver.azureedge.net/LATEST_%s" % channel.upper() 943 version = get(version_url).text.strip() 944 945 if self.platform == "macos": 946 bits = "mac64" 947 edgedriver_path = os.path.join(dest, self.edgedriver_name) 948 else: 949 bits = "win64" if uname[4] == "x86_64" else "win32" 950 edgedriver_path = os.path.join(dest, "%s.exe" % self.edgedriver_name) 951 url = "https://msedgedriver.azureedge.net/%s/edgedriver_%s.zip" % (version, bits) 952 953 # cleanup existing Edge driver files to avoid access_denied errors when unzipping 954 if os.path.isfile(edgedriver_path): 955 # remove read-only attribute 956 os.chmod(edgedriver_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 957 print("Delete %s file" % edgedriver_path) 958 os.remove(edgedriver_path) 959 driver_notes_path = os.path.join(dest, "Driver_notes") 960 if os.path.isdir(driver_notes_path): 961 print("Delete %s folder" % driver_notes_path) 962 shutil.rmtree(driver_notes_path, ignore_errors=False, onerror=handle_remove_readonly) 963 964 self.logger.info("Downloading MSEdgeDriver from %s" % url) 965 unzip(get(url).raw, dest) 966 if os.path.isfile(edgedriver_path): 967 self.logger.info("Successfully downloaded MSEdgeDriver to %s" % edgedriver_path) 968 return find_executable(self.edgedriver_name, dest) 969 970 def version(self, binary=None, webdriver_binary=None): 971 if binary is None: 972 binary = self.find_binary() 973 if self.platform != "win": 974 try: 975 version_string = call(binary, "--version").strip() 976 except subprocess.CalledProcessError: 977 self.logger.warning("Failed to call %s" % binary) 978 return None 979 m = re.match(r"Microsoft Edge (.*) ", version_string) 980 if not m: 981 self.logger.warning("Failed to extract version from: %s" % version_string) 982 return None 983 return m.group(1) 984 else: 985 if binary is not None: 986 return _get_fileversion(binary, self.logger) 987 self.logger.warning("Failed to find Edge binary.") 988 return None 989 990 991class Edge(Browser): 992 """Edge-specific interface.""" 993 994 product = "edge" 995 requirements = "requirements_edge.txt" 996 997 def download(self, dest=None, channel=None, rename=None): 998 raise NotImplementedError 999 1000 def install(self, dest=None, channel=None): 1001 raise NotImplementedError 1002 1003 def find_binary(self, venv_path=None, channel=None): 1004 raise NotImplementedError 1005 1006 def find_webdriver(self, channel=None): 1007 return find_executable("MicrosoftWebDriver") 1008 1009 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1010 raise NotImplementedError 1011 1012 def version(self, binary=None, webdriver_binary=None): 1013 command = "(Get-AppxPackage Microsoft.MicrosoftEdge).Version" 1014 try: 1015 return call("powershell.exe", command).strip() 1016 except (subprocess.CalledProcessError, OSError): 1017 self.logger.warning("Failed to call %s in PowerShell" % command) 1018 return None 1019 1020 1021class EdgeWebDriver(Edge): 1022 product = "edge_webdriver" 1023 1024 1025class InternetExplorer(Browser): 1026 """Internet Explorer-specific interface.""" 1027 1028 product = "ie" 1029 requirements = "requirements_ie.txt" 1030 1031 def download(self, dest=None, channel=None, rename=None): 1032 raise NotImplementedError 1033 1034 def install(self, dest=None, channel=None): 1035 raise NotImplementedError 1036 1037 def find_binary(self, venv_path=None, channel=None): 1038 raise NotImplementedError 1039 1040 def find_webdriver(self, channel=None): 1041 return find_executable("IEDriverServer.exe") 1042 1043 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1044 raise NotImplementedError 1045 1046 def version(self, binary=None, webdriver_binary=None): 1047 return None 1048 1049 1050class Safari(Browser): 1051 """Safari-specific interface. 1052 1053 Includes installation, webdriver installation, and wptrunner setup methods. 1054 """ 1055 1056 product = "safari" 1057 requirements = "requirements_safari.txt" 1058 1059 def download(self, dest=None, channel=None, rename=None): 1060 raise NotImplementedError 1061 1062 def install(self, dest=None, channel=None): 1063 raise NotImplementedError 1064 1065 def find_binary(self, venv_path=None, channel=None): 1066 raise NotImplementedError 1067 1068 def find_webdriver(self, channel=None): 1069 path = None 1070 if channel == "preview": 1071 path = "/Applications/Safari Technology Preview.app/Contents/MacOS" 1072 return find_executable("safaridriver", path) 1073 1074 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1075 raise NotImplementedError 1076 1077 def version(self, binary=None, webdriver_binary=None): 1078 if webdriver_binary is None: 1079 self.logger.warning("Cannot find Safari version without safaridriver") 1080 return None 1081 # Use `safaridriver --version` to get the version. Example output: 1082 # "Included with Safari 12.1 (14607.1.11)" 1083 # "Included with Safari Technology Preview (Release 67, 13607.1.9.0.1)" 1084 # The `--version` flag was added in STP 67, so allow the call to fail. 1085 try: 1086 version_string = call(webdriver_binary, "--version").strip() 1087 except subprocess.CalledProcessError: 1088 self.logger.warning("Failed to call %s --version" % webdriver_binary) 1089 return None 1090 m = re.match(br"Included with Safari (.*)", version_string) 1091 if not m: 1092 self.logger.warning("Failed to extract version from: %s" % version_string) 1093 return None 1094 return m.group(1).decode() 1095 1096 1097class Servo(Browser): 1098 """Servo-specific interface.""" 1099 1100 product = "servo" 1101 requirements = "requirements_servo.txt" 1102 1103 def platform_components(self): 1104 platform = { 1105 "Linux": "linux", 1106 "Windows": "win", 1107 "Darwin": "mac" 1108 }.get(uname[0]) 1109 1110 if platform is None: 1111 raise ValueError("Unable to construct a valid Servo package for current platform") 1112 1113 if platform == "linux": 1114 extension = ".tar.gz" 1115 decompress = untar 1116 elif platform == "win" or platform == "mac": 1117 raise ValueError("Unable to construct a valid Servo package for current platform") 1118 1119 return (platform, extension, decompress) 1120 1121 def _get(self, channel="nightly"): 1122 if channel != "nightly": 1123 raise ValueError("Only nightly versions of Servo are available") 1124 1125 platform, extension, _ = self.platform_components() 1126 url = "https://download.servo.org/nightly/%s/servo-latest%s" % (platform, extension) 1127 return get(url) 1128 1129 def download(self, dest=None, channel="nightly", rename=None): 1130 if dest is None: 1131 dest = os.pwd 1132 1133 resp = self._get(dest, channel) 1134 _, extension, _ = self.platform_components() 1135 1136 filename = rename if rename is not None else "servo-latest" 1137 with open(os.path.join(dest, "%s%s" % (filename, extension,)), "w") as f: 1138 f.write(resp.content) 1139 1140 def install(self, dest=None, channel="nightly"): 1141 """Install latest Browser Engine.""" 1142 if dest is None: 1143 dest = os.pwd 1144 1145 _, _, decompress = self.platform_components() 1146 1147 resp = self._get(channel) 1148 decompress(resp.raw, dest=dest) 1149 path = find_executable("servo", os.path.join(dest, "servo")) 1150 st = os.stat(path) 1151 os.chmod(path, st.st_mode | stat.S_IEXEC) 1152 return path 1153 1154 def find_binary(self, venv_path=None, channel=None): 1155 path = find_executable("servo", os.path.join(venv_path, "servo")) 1156 if path is None: 1157 path = find_executable("servo") 1158 return path 1159 1160 def find_webdriver(self, channel=None): 1161 return None 1162 1163 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1164 raise NotImplementedError 1165 1166 def version(self, binary=None, webdriver_binary=None): 1167 """Retrieve the release version of the installed browser.""" 1168 output = call(binary, "--version") 1169 m = re.search(r"Servo ([0-9\.]+-[a-f0-9]+)?(-dirty)?$", output.strip()) 1170 if m: 1171 return m.group(0) 1172 1173 1174class ServoWebDriver(Servo): 1175 product = "servodriver" 1176 1177 1178class Sauce(Browser): 1179 """Sauce-specific interface.""" 1180 1181 product = "sauce" 1182 requirements = "requirements_sauce.txt" 1183 1184 def download(self, dest=None, channel=None, rename=None): 1185 raise NotImplementedError 1186 1187 def install(self, dest=None, channel=None): 1188 raise NotImplementedError 1189 1190 def find_binary(self, venev_path=None, channel=None): 1191 raise NotImplementedError 1192 1193 def find_webdriver(self, channel=None): 1194 raise NotImplementedError 1195 1196 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1197 raise NotImplementedError 1198 1199 def version(self, binary=None, webdriver_binary=None): 1200 return None 1201 1202 1203class WebKit(Browser): 1204 """WebKit-specific interface.""" 1205 1206 product = "webkit" 1207 requirements = "requirements_webkit.txt" 1208 1209 def download(self, dest=None, channel=None, rename=None): 1210 raise NotImplementedError 1211 1212 def install(self, dest=None, channel=None): 1213 raise NotImplementedError 1214 1215 def find_binary(self, venv_path=None, channel=None): 1216 return None 1217 1218 def find_webdriver(self, channel=None): 1219 return None 1220 1221 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1222 raise NotImplementedError 1223 1224 def version(self, binary=None, webdriver_binary=None): 1225 return None 1226 1227 1228class WebKitGTKMiniBrowser(WebKit): 1229 1230 def find_binary(self, venv_path=None, channel=None): 1231 libexecpaths = ["/usr/libexec/webkit2gtk-4.0"] # Fedora path 1232 triplet = "x86_64-linux-gnu" 1233 # Try to use GCC to detect this machine triplet 1234 gcc = find_executable("gcc") 1235 if gcc: 1236 try: 1237 triplet = call(gcc, "-dumpmachine").decode().strip() 1238 except subprocess.CalledProcessError: 1239 pass 1240 # Add Debian/Ubuntu path 1241 libexecpaths.append("/usr/lib/%s/webkit2gtk-4.0" % triplet) 1242 if channel == "nightly": 1243 libexecpaths.append("/opt/webkitgtk/nightly") 1244 return find_executable("MiniBrowser", os.pathsep.join(libexecpaths)) 1245 1246 def find_webdriver(self, channel=None): 1247 path = os.environ['PATH'] 1248 if channel == "nightly": 1249 path = "%s:%s" % (path, "/opt/webkitgtk/nightly") 1250 return find_executable("WebKitWebDriver", path) 1251 1252 def version(self, binary=None, webdriver_binary=None): 1253 if binary is None: 1254 return None 1255 try: # WebKitGTK MiniBrowser before 2.26.0 doesn't support --version 1256 output = call(binary, "--version").decode().strip() 1257 except subprocess.CalledProcessError: 1258 return None 1259 # Example output: "WebKitGTK 2.26.1" 1260 if output: 1261 m = re.match(r"WebKitGTK (.+)", output) 1262 if not m: 1263 self.logger.warning("Failed to extract version from: %s" % output) 1264 return None 1265 return m.group(1) 1266 return None 1267 1268 1269class Epiphany(Browser): 1270 """Epiphany-specific interface.""" 1271 1272 product = "epiphany" 1273 requirements = "requirements_epiphany.txt" 1274 1275 def download(self, dest=None, channel=None, rename=None): 1276 raise NotImplementedError 1277 1278 def install(self, dest=None, channel=None): 1279 raise NotImplementedError 1280 1281 def find_binary(self, venv_path=None, channel=None): 1282 return find_executable("epiphany") 1283 1284 def find_webdriver(self, channel=None): 1285 return find_executable("WebKitWebDriver") 1286 1287 def install_webdriver(self, dest=None, channel=None, browser_binary=None): 1288 raise NotImplementedError 1289 1290 def version(self, binary=None, webdriver_binary=None): 1291 if binary is None: 1292 return None 1293 output = call(binary, "--version") 1294 if output: 1295 # Stable release output looks like: "Web 3.30.2" 1296 # Tech Preview output looks like "Web 3.31.3-88-g97db4f40f" 1297 return output.split()[1] 1298 return None 1299