1import os
2import platform
3import re
4import shutil
5import stat
6import errno
7import subprocess
8import tempfile
9from abc import ABCMeta, abstractmethod
10from datetime import datetime, timedelta
11from distutils.spawn import find_executable
12
13from six.moves.urllib.parse import urlsplit
14import requests
15
16from .utils import call, get, untar, unzip
17
18uname = platform.uname()
19
20# the rootUrl for the firefox-ci deployment of Taskcluster
21# (after November 9, https://firefox-ci-tc.services.mozilla.com/)
22FIREFOX_CI_ROOT_URL = 'https://taskcluster.net'
23
24
25def _get_fileversion(binary, logger=None):
26    command = "(Get-Item '%s').VersionInfo.FileVersion" % binary.replace("'", "''")
27    try:
28        return call("powershell.exe", command).strip()
29    except (subprocess.CalledProcessError, OSError):
30        if logger is not None:
31            logger.warning("Failed to call %s in PowerShell" % command)
32        return None
33
34
35def handle_remove_readonly(func, path, exc):
36    excvalue = exc[1]
37    if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
38        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777
39        func(path)
40    else:
41        raise
42
43
44def get_ext(filename):
45    """Get the extension from a filename with special handling for .tar.foo"""
46    name, ext = os.path.splitext(filename)
47    if name.endswith(".tar"):
48        ext = ".tar%s" % ext
49    return ext
50
51
52class Browser(object):
53    __metaclass__ = ABCMeta
54
55    def __init__(self, logger):
56        self.logger = logger
57
58    @abstractmethod
59    def download(self, dest=None, channel=None, rename=None):
60        """Download a package or installer for the browser
61        :param dest: Directory in which to put the dowloaded package
62        :param channel: Browser channel to download
63        :param rename: Optional name for the downloaded package; the original
64                       extension is preserved.
65        """
66        return NotImplemented
67
68    @abstractmethod
69    def install(self, dest=None):
70        """Install the browser."""
71        return NotImplemented
72
73    @abstractmethod
74    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
75        """Install the WebDriver implementation for this browser."""
76        return NotImplemented
77
78    @abstractmethod
79    def find_binary(self, venv_path=None, channel=None):
80        """Find the binary of the browser.
81
82        If the WebDriver for the browser is able to find the binary itself, this
83        method doesn't need to be implemented, in which case NotImplementedError
84        is suggested to be raised to prevent accidental use.
85        """
86        return NotImplemented
87
88    @abstractmethod
89    def find_webdriver(self, channel=None):
90        """Find the binary of the WebDriver."""
91        return NotImplemented
92
93    @abstractmethod
94    def version(self, binary=None, webdriver_binary=None):
95        """Retrieve the release version of the installed browser."""
96        return NotImplemented
97
98    @abstractmethod
99    def requirements(self):
100        """Name of the browser-specific wptrunner requirements file"""
101        return NotImplemented
102
103
104class Firefox(Browser):
105    """Firefox-specific interface.
106
107    Includes installation, webdriver installation, and wptrunner setup methods.
108    """
109
110    product = "firefox"
111    binary = "browsers/firefox/firefox"
112    requirements = "requirements_firefox.txt"
113
114    platform = {
115        "Linux": "linux",
116        "Windows": "win",
117        "Darwin": "macos"
118    }.get(uname[0])
119
120    application_name = {
121        "stable": "Firefox.app",
122        "beta": "Firefox.app",
123        "nightly": "Firefox Nightly.app"
124    }
125
126    def platform_string_geckodriver(self):
127        if self.platform is None:
128            raise ValueError("Unable to construct a valid Geckodriver package name for current platform")
129
130        if self.platform in ("linux", "win"):
131            bits = "64" if uname[4] == "x86_64" else "32"
132        else:
133            bits = ""
134
135        return "%s%s" % (self.platform, bits)
136
137    def _get_dest(self, dest, channel):
138        if dest is None:
139            # os.getcwd() doesn't include the venv path
140            dest = os.path.join(os.getcwd(), "_venv")
141
142        dest = os.path.join(dest, "browsers", channel)
143
144        if not os.path.exists(dest):
145            os.makedirs(dest)
146
147        return dest
148
149    def download(self, dest=None, channel="nightly", rename=None):
150        product = {
151            "nightly": "firefox-nightly-latest-ssl",
152            "beta": "firefox-beta-latest-ssl",
153            "stable": "firefox-latest-ssl"
154        }
155
156        os_builds = {
157            ("linux", "x86"): "linux",
158            ("linux", "x86_64"): "linux64",
159            ("win", "x86"): "win",
160            ("win", "AMD64"): "win64",
161            ("macos", "x86_64"): "osx",
162        }
163        os_key = (self.platform, uname[4])
164
165        if dest is None:
166            dest = self._get_dest(None, channel)
167
168        if channel not in product:
169            raise ValueError("Unrecognised release channel: %s" % channel)
170
171        if os_key not in os_builds:
172            raise ValueError("Unsupported platform: %s %s" % os_key)
173
174        url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel],
175                                                                             os_builds[os_key])
176        self.logger.info("Downloading Firefox from %s" % url)
177        resp = requests.get(url)
178
179        filename = None
180
181        content_disposition = resp.headers.get('content-disposition')
182        if content_disposition:
183            filenames = re.findall("filename=(.+)", content_disposition)
184            if filenames:
185                filename = filenames[0]
186
187        if not filename:
188            filename = urlsplit(resp.url).path.rsplit("/", 1)[1]
189
190        if not filename:
191            filename = "firefox.tar.bz2"
192
193        if rename:
194            filename = "%s%s" % (rename, get_ext(filename))
195
196        installer_path = os.path.join(dest, filename)
197
198        with open(installer_path, "wb") as f:
199            f.write(resp.content)
200
201        return installer_path
202
203    def install(self, dest=None, channel="nightly"):
204        """Install Firefox."""
205        import mozinstall
206
207        dest = self._get_dest(dest, channel)
208
209        filename = os.path.basename(dest)
210
211        installer_path = self.download(dest, channel)
212
213        try:
214            mozinstall.install(installer_path, dest)
215        except mozinstall.mozinstall.InstallError:
216            if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))):
217                # mozinstall will fail if nightly is already installed in the venv because
218                # mac installation uses shutil.copy_tree
219                mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app")))
220                mozinstall.install(filename, dest)
221            else:
222                raise
223
224        os.remove(installer_path)
225        return self.find_binary_path(dest)
226
227    def find_binary_path(self, path=None, channel="nightly"):
228        """Looks for the firefox binary in the virtual environment"""
229
230        if path is None:
231            # os.getcwd() doesn't include the venv path
232            path = os.path.join(os.getcwd(), "_venv", "browsers", channel)
233
234        binary = None
235
236        if self.platform == "linux":
237            binary = find_executable("firefox", os.path.join(path, "firefox"))
238        elif self.platform == "win":
239            import mozinstall
240            try:
241                binary = mozinstall.get_binary(path, "firefox")
242            except mozinstall.InvalidBinary:
243                # ignore the case where we fail to get a binary
244                pass
245        elif self.platform == "macos":
246            binary = find_executable("firefox", os.path.join(path, self.application_name.get(channel, "Firefox Nightly.app"),
247                                                             "Contents", "MacOS"))
248
249        return binary
250
251    def find_binary(self, venv_path=None, channel="nightly"):
252        if venv_path is None:
253            venv_path = os.path.join(os.getcwd(), "_venv")
254
255        path = os.path.join(venv_path, "browsers", channel)
256        binary = self.find_binary_path(path, channel)
257
258        if not binary and self.platform == "win":
259            winpaths = [os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Mozilla Firefox"),
260                        os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Mozilla Firefox")]
261            for winpath in winpaths:
262                binary = self.find_binary_path(winpath, channel)
263                if binary is not None:
264                    break
265
266        if not binary and self.platform == "macos":
267            macpaths = ["/Applications/Firefox Nightly.app/Contents/MacOS",
268                        os.path.expanduser("~/Applications/Firefox Nightly.app/Contents/MacOS"),
269                        "/Applications/Firefox Developer Edition.app/Contents/MacOS",
270                        os.path.expanduser("~/Applications/Firefox Developer Edition.app/Contents/MacOS"),
271                        "/Applications/Firefox.app/Contents/MacOS",
272                        os.path.expanduser("~/Applications/Firefox.app/Contents/MacOS")]
273            return find_executable("firefox", os.pathsep.join(macpaths))
274
275        if binary is None:
276            return find_executable("firefox")
277
278        return binary
279
280    def find_certutil(self):
281        path = find_executable("certutil")
282        if path is None:
283            return None
284        if os.path.splitdrive(os.path.normcase(path))[1].split(os.path.sep) == ["", "windows", "system32", "certutil.exe"]:
285            return None
286        return path
287
288    def find_webdriver(self, channel=None):
289        return find_executable("geckodriver")
290
291    def get_version_and_channel(self, binary):
292        version_string = call(binary, "--version").strip()
293        m = re.match(r"Mozilla Firefox (\d+\.\d+(?:\.\d+)?)(a|b)?", version_string)
294        if not m:
295            return None, "nightly"
296        version, status = m.groups()
297        channel = {"a": "nightly", "b": "beta"}
298        return version, channel.get(status, "stable")
299
300    def get_profile_bundle_url(self, version, channel):
301        if channel == "stable":
302            repo = "https://hg.mozilla.org/releases/mozilla-release"
303            tag = "FIREFOX_%s_RELEASE" % version.replace(".", "_")
304        elif channel == "beta":
305            repo = "https://hg.mozilla.org/releases/mozilla-beta"
306            major_version = version.split(".", 1)[0]
307            # For beta we have a different format for betas that are now in stable releases
308            # vs those that are not
309            tags = get("https://hg.mozilla.org/releases/mozilla-beta/json-tags").json()["tags"]
310            tags = {item["tag"] for item in tags}
311            end_tag = "FIREFOX_BETA_%s_END" % major_version
312            if end_tag in tags:
313                tag = end_tag
314            else:
315                tag = "tip"
316        else:
317            repo = "https://hg.mozilla.org/mozilla-central"
318            # Always use tip as the tag for nightly; this isn't quite right
319            # but to do better we need the actual build revision, which we
320            # can get if we have an application.ini file
321            tag = "tip"
322
323        return "%s/archive/%s.zip/testing/profiles/" % (repo, tag)
324
325    def install_prefs(self, binary, dest=None, channel=None):
326        if binary:
327            version, channel_ = self.get_version_and_channel(binary)
328            if channel is not None and channel != channel_:
329                # Beta doesn't always seem to have the b in the version string, so allow the
330                # manually supplied value to override the one from the binary
331                self.logger.warning("Supplied channel doesn't match binary, using supplied channel")
332            elif channel is None:
333                channel = channel_
334        else:
335            version = None
336
337        if dest is None:
338            dest = os.curdir
339
340        dest = os.path.join(dest, "profiles", channel)
341        if version:
342            dest = os.path.join(dest, version)
343        have_cache = False
344        if os.path.exists(dest) and len(os.listdir(dest)) > 0:
345            if channel != "nightly":
346                have_cache = True
347            else:
348                now = datetime.now()
349                have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
350                              now - timedelta(days=1))
351
352        # If we don't have a recent download, grab and extract the latest one
353        if not have_cache:
354            if os.path.exists(dest):
355                shutil.rmtree(dest)
356            os.makedirs(dest)
357
358            url = self.get_profile_bundle_url(version, channel)
359
360            self.logger.info("Installing test prefs from %s" % url)
361            try:
362                extract_dir = tempfile.mkdtemp()
363                unzip(get(url).raw, dest=extract_dir)
364
365                profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
366                for name in os.listdir(profiles):
367                    path = os.path.join(profiles, name)
368                    shutil.move(path, dest)
369            finally:
370                shutil.rmtree(extract_dir)
371        else:
372            self.logger.info("Using cached test prefs from %s" % dest)
373
374        return dest
375
376    def _latest_geckodriver_version(self):
377        """Get and return latest version number for geckodriver."""
378        # This is used rather than an API call to avoid rate limits
379        tags = call("git", "ls-remote", "--tags", "--refs",
380                    "https://github.com/mozilla/geckodriver.git")
381        release_re = re.compile(r".*refs/tags/v(\d+)\.(\d+)\.(\d+)")
382        latest_release = 0
383        for item in tags.split("\n"):
384            m = release_re.match(item)
385            if m:
386                version = [int(item) for item in m.groups()]
387                if version > latest_release:
388                    latest_release = version
389        assert latest_release != 0
390        return "v%s.%s.%s" % tuple(str(item) for item in latest_release)
391
392    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
393        """Install latest Geckodriver."""
394        if dest is None:
395            dest = os.getcwd()
396
397        if channel == "nightly":
398            path = self.install_geckodriver_nightly(dest)
399            if path is not None:
400                return path
401            else:
402                self.logger.warning("Nightly webdriver not found; falling back to release")
403
404        version = self._latest_geckodriver_version()
405        format = "zip" if uname[0] == "Windows" else "tar.gz"
406        self.logger.debug("Latest geckodriver release %s" % version)
407        url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" %
408               (version, version, self.platform_string_geckodriver(), format))
409        if format == "zip":
410            unzip(get(url).raw, dest=dest)
411        else:
412            untar(get(url).raw, dest=dest)
413        return find_executable(os.path.join(dest, "geckodriver"))
414
415    def install_geckodriver_nightly(self, dest):
416        import tarfile
417        import mozdownload
418        self.logger.info("Attempting to install webdriver from nightly")
419        try:
420            s = mozdownload.DailyScraper(branch="mozilla-central",
421                                         extension="common.tests.tar.gz",
422                                         destination=dest)
423            package_path = s.download()
424        except mozdownload.errors.NotFoundError:
425            return
426
427        try:
428            exe_suffix = ".exe" if uname[0] == "Windows" else ""
429            with tarfile.open(package_path, "r") as f:
430                try:
431                    member = f.getmember("bin%sgeckodriver%s" % (os.path.sep,
432                                                                 exe_suffix))
433                except KeyError:
434                    return
435                # Remove bin/ from the path.
436                member.name = os.path.basename(member.name)
437                f.extractall(members=[member], path=dest)
438                path = os.path.join(dest, member.name)
439            self.logger.info("Extracted geckodriver to %s" % path)
440        finally:
441            os.unlink(package_path)
442
443        return path
444
445    def version(self, binary=None, webdriver_binary=None):
446        """Retrieve the release version of the installed browser."""
447        version_string = call(binary, "--version").strip()
448        m = re.match(r"Mozilla Firefox (.*)", version_string)
449        if not m:
450            return None
451        return m.group(1)
452
453
454class FirefoxAndroid(Browser):
455    """Android-specific Firefox interface."""
456
457    product = "firefox_android"
458    requirements = "requirements_firefox.txt"
459
460    def download(self, dest=None, channel=None, rename=None):
461        if dest is None:
462            dest = os.pwd
463
464        if FIREFOX_CI_ROOT_URL == 'https://taskcluster.net':
465            # NOTE: this condition can be removed after November 9, 2019
466            TC_QUEUE_BASE = "https://queue.taskcluster.net/v1/"
467            TC_INDEX_BASE = "https://index.taskcluster.net/v1/"
468        else:
469            TC_QUEUE_BASE = FIREFOX_CI_ROOT_URL + "/api/queue/v1/"
470            TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/"
471
472
473        resp = requests.get(TC_INDEX_BASE +
474                            "task/gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt")
475        resp.raise_for_status()
476        index = resp.json()
477        task_id = index["taskId"]
478
479        resp = requests.get(TC_QUEUE_BASE + "task/%s/artifacts/%s" %
480                            (task_id, "public/build/geckoview-androidTest.apk"))
481        resp.raise_for_status()
482
483        filename = "geckoview-androidTest.apk"
484        if rename:
485            filename = "%s%s" % (rename, get_ext(filename)[1])
486        apk_path = os.path.join(dest, filename)
487
488        with open(apk_path, "wb") as f:
489            f.write(resp.content)
490
491        return apk_path
492
493    def install(self, dest=None, channel=None):
494        return self.download(dest, channel)
495
496    def install_prefs(self, binary, dest=None, channel=None):
497        fx_browser = Firefox(self.logger)
498        return fx_browser.install_prefs(binary, dest, channel)
499
500    def find_binary(self, venv_path=None, channel=None):
501        raise NotImplementedError
502
503    def find_webdriver(self, channel=None):
504        raise NotImplementedError
505
506    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
507        raise NotImplementedError
508
509    def version(self, binary=None, webdriver_binary=None):
510        return None
511
512
513class Chrome(Browser):
514    """Chrome-specific interface.
515
516    Includes webdriver installation, and wptrunner setup methods.
517    """
518
519    product = "chrome"
520    requirements = "requirements_chrome.txt"
521
522    def download(self, dest=None, channel=None, rename=None):
523        raise NotImplementedError
524
525    def install(self, dest=None, channel=None):
526        raise NotImplementedError
527
528    def platform_string(self):
529        platform = {
530            "Linux": "linux",
531            "Windows": "win",
532            "Darwin": "mac"
533        }.get(uname[0])
534
535        if platform is None:
536            raise ValueError("Unable to construct a valid Chrome package name for current platform")
537
538        if platform == "linux":
539            bits = "64" if uname[4] == "x86_64" else "32"
540        elif platform == "mac":
541            bits = "64"
542        elif platform == "win":
543            bits = "32"
544
545        return "%s%s" % (platform, bits)
546
547    def chromium_platform_string(self):
548        platform = {
549            "Linux": "Linux",
550            "Windows": "Win",
551            "Darwin": "Mac"
552        }.get(uname[0])
553
554        if platform is None:
555            raise ValueError("Unable to construct a valid Chromium package name for current platform")
556
557        if (platform == "Linux" or platform == "Win") and uname[4] == "x86_64":
558            platform += "_x64"
559
560        return platform
561
562    def find_binary(self, venv_path=None, channel=None):
563        if uname[0] == "Linux":
564            name = "google-chrome"
565            if channel == "stable":
566                name += "-stable"
567            elif channel == "beta":
568                name += "-beta"
569            elif channel == "dev":
570                name += "-unstable"
571            # No Canary on Linux.
572            return find_executable(name)
573        if uname[0] == "Darwin":
574            if channel == "canary":
575                return "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"
576            # All other channels share the same path on macOS.
577            return "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
578        if uname[0] == "Windows":
579            return os.path.expandvars(r"$SYSTEMDRIVE\Program Files (x86)\Google\Chrome\Application\chrome.exe")
580        self.logger.warning("Unable to find the browser binary.")
581        return None
582
583    def find_webdriver(self, channel=None):
584        return find_executable("chromedriver")
585
586    def _official_chromedriver_url(self, chrome_version):
587        # http://chromedriver.chromium.org/downloads/version-selection
588        parts = chrome_version.split(".")
589        assert len(parts) == 4
590        latest_url = "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_%s.%s.%s" % tuple(parts[:-1])
591        try:
592            latest = get(latest_url).text.strip()
593        except requests.RequestException:
594            latest_url = "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_%s" % parts[0]
595            try:
596                latest = get(latest_url).text.strip()
597            except requests.RequestException:
598                return None
599        return "https://chromedriver.storage.googleapis.com/%s/chromedriver_%s.zip" % (
600            latest, self.platform_string())
601
602    def _chromium_chromedriver_url(self, chrome_version):
603        try:
604            # Try to find the Chromium build with the same revision.
605            omaha = get("https://omahaproxy.appspot.com/deps.json?version=" + chrome_version).json()
606            revision = omaha['chromium_base_position']
607            url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/%s/chromedriver_%s.zip" % (
608                self.chromium_platform_string(), revision, self.platform_string())
609            # Check the status without downloading the content (this is a streaming request).
610            get(url)
611        except requests.RequestException:
612            # Fall back to the tip-of-tree Chromium build.
613            revision_url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/LAST_CHANGE" % (
614                self.chromium_platform_string())
615            revision = get(revision_url).text.strip()
616            url = "https://storage.googleapis.com/chromium-browser-snapshots/%s/%s/chromedriver_%s.zip" % (
617                self.chromium_platform_string(), revision, self.platform_string())
618        return url
619
620    def _latest_chromedriver_url(self, chrome_version):
621        # Remove channel suffixes (e.g. " dev").
622        chrome_version = chrome_version.split(' ')[0]
623        return (self._official_chromedriver_url(chrome_version) or
624                self._chromium_chromedriver_url(chrome_version))
625
626    def install_webdriver_by_version(self, version, dest=None):
627        assert version, "Cannot install ChromeDriver without Chrome version"
628        if dest is None:
629            dest = os.pwd
630        url = self._latest_chromedriver_url(version)
631        self.logger.info("Downloading ChromeDriver from %s" % url)
632        unzip(get(url).raw, dest)
633        chromedriver_dir = os.path.join(
634            dest, 'chromedriver_%s' % self.platform_string())
635        if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")):
636            shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest)
637            shutil.rmtree(chromedriver_dir)
638        return find_executable("chromedriver", dest)
639
640    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
641        if browser_binary is None:
642            browser_binary = self.find_binary(channel)
643        return self.install_webdriver_by_version(
644            self.version(browser_binary), dest)
645
646    def version(self, binary=None, webdriver_binary=None):
647        if not binary:
648            self.logger.warning("No browser binary provided.")
649            return None
650
651        if uname[0] == "Windows":
652            return _get_fileversion(binary, self.logger)
653
654        try:
655            version_string = call(binary, "--version").strip()
656        except subprocess.CalledProcessError:
657            self.logger.warning("Failed to call %s" % binary)
658            return None
659        m = re.match(r"(?:Google Chrome|Chromium) (.*)", version_string)
660        if not m:
661            self.logger.warning("Failed to extract version from: %s" % version_string)
662            return None
663        return m.group(1)
664
665
666class ChromeAndroidBase(Browser):
667    """A base class for ChromeAndroid and AndroidWebView.
668
669    On Android, WebView is based on Chromium open source project, and on some
670    versions of Android we share the library with Chrome. Therefore, we have
671    a very similar WPT runner implementation.
672    Includes webdriver installation.
673    """
674    __metaclass__ = ABCMeta  # This is an abstract class.
675
676    def __init__(self, logger):
677        super(ChromeAndroidBase, self).__init__(logger)
678        self.device_serial = None
679
680    def download(self, dest=None, channel=None, rename=None):
681        raise NotImplementedError
682
683    def install(self, dest=None, channel=None):
684        raise NotImplementedError
685
686    @abstractmethod
687    def find_binary(self, venv_path=None, channel=None):
688        raise NotImplementedError
689
690    def find_webdriver(self, channel=None):
691        return find_executable("chromedriver")
692
693    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
694        if browser_binary is None:
695            browser_binary = self.find_binary(channel)
696        chrome = Chrome(self.logger)
697        return chrome.install_webdriver_by_version(
698            self.version(browser_binary), dest)
699
700    def version(self, binary=None, webdriver_binary=None):
701        if not binary:
702            self.logger.warning("No package name provided.")
703            return None
704
705        command = ['adb']
706        if self.device_serial:
707            command.extend(['-s', self.device_serial])
708        command.extend(['shell', 'dumpsys', 'package', binary])
709        try:
710            output = call(*command)
711        except (subprocess.CalledProcessError, OSError):
712            self.logger.warning("Failed to call %s" % " ".join(command))
713            return None
714        match = re.search(r'versionName=(.*)', output)
715        if not match:
716            self.logger.warning("Failed to find versionName")
717            return None
718        return match.group(1)
719
720
721class ChromeAndroid(ChromeAndroidBase):
722    """Chrome-specific interface for Android.
723    """
724
725    product = "chrome_android"
726    requirements = "requirements_chrome_android.txt"
727
728    def find_binary(self, venv_path=None, channel=None):
729        if channel in ("beta", "dev", "canary"):
730            return "com.chrome." + channel
731        return "com.android.chrome"
732
733
734#TODO(aluo): This is largely copied from the AndroidWebView implementation.
735#            Tests are not running for weblayer yet (crbug/1019521), this
736#            initial implementation will help to reproduce and debug any issues.
737class AndroidWeblayer(ChromeAndroidBase):
738    """Weblayer-specific interface for Android."""
739
740    product = "android_weblayer"
741    # TODO(aluo): replace this with weblayer version after tests are working.
742    requirements = "requirements_android_webview.txt"
743
744    def find_binary(self, venv_path=None, channel=None):
745        return "org.chromium.weblayer.shell"
746
747
748class AndroidWebview(ChromeAndroidBase):
749    """Webview-specific interface for Android.
750
751    Design doc:
752    https://docs.google.com/document/d/19cGz31lzCBdpbtSC92svXlhlhn68hrsVwSB7cfZt54o/view
753    """
754
755    product = "android_webview"
756    requirements = "requirements_android_webview.txt"
757
758    def find_binary(self, venv_path=None, channel=None):
759        # Just get the current package name of the WebView provider.
760        # For WebView, it is not trivial to change the WebView provider, so
761        # we will just grab whatever is available.
762        # https://chromium.googlesource.com/chromium/src/+/HEAD/android_webview/docs/channels.md
763        command = ['adb']
764        if self.device_serial:
765            command.extend(['-s', self.device_serial])
766        command.extend(['shell', 'dumpsys', 'webviewupdate'])
767        try:
768            output = call(*command)
769        except (subprocess.CalledProcessError, OSError):
770            self.logger.warning("Failed to call %s" % " ".join(command))
771            return None
772        m = re.search(r'^\s*Current WebView package \(name, version\): \((.*), ([0-9.]*)\)$',
773                      output, re.M)
774        if m is None:
775            self.logger.warning("Unable to find current WebView package in dumpsys output")
776            return None
777        self.logger.warning("Final package name: " + m.group(1))
778        return m.group(1)
779
780
781class ChromeiOS(Browser):
782    """Chrome-specific interface for iOS.
783    """
784
785    product = "chrome_ios"
786    requirements = "requirements_chrome_ios.txt"
787
788    def download(self, dest=None, channel=None, rename=None):
789        raise NotImplementedError
790
791    def install(self, dest=None, channel=None):
792        raise NotImplementedError
793
794    def find_binary(self, venv_path=None, channel=None):
795        raise NotImplementedError
796
797    def find_webdriver(self, channel=None):
798        raise NotImplementedError
799
800    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
801        raise NotImplementedError
802
803    def version(self, binary=None, webdriver_binary=None):
804        return None
805
806
807class Opera(Browser):
808    """Opera-specific interface.
809
810    Includes webdriver installation, and wptrunner setup methods.
811    """
812
813    product = "opera"
814    requirements = "requirements_opera.txt"
815
816    @property
817    def binary(self):
818        if uname[0] == "Linux":
819            return "/usr/bin/opera"
820        # TODO Windows, Mac?
821        self.logger.warning("Unable to find the browser binary.")
822        return None
823
824    def download(self, dest=None, channel=None, rename=None):
825        raise NotImplementedError
826
827    def install(self, dest=None, channel=None):
828        raise NotImplementedError
829
830    def platform_string(self):
831        platform = {
832            "Linux": "linux",
833            "Windows": "win",
834            "Darwin": "mac"
835        }.get(uname[0])
836
837        if platform is None:
838            raise ValueError("Unable to construct a valid Opera package name for current platform")
839
840        if platform == "linux":
841            bits = "64" if uname[4] == "x86_64" else "32"
842        elif platform == "mac":
843            bits = "64"
844        elif platform == "win":
845            bits = "32"
846
847        return "%s%s" % (platform, bits)
848
849    def find_binary(self, venv_path=None, channel=None):
850        raise NotImplementedError
851
852    def find_webdriver(self, channel=None):
853        return find_executable("operadriver")
854
855    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
856        if dest is None:
857            dest = os.pwd
858        latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"]
859        url = "https://github.com/operasoftware/operachromiumdriver/releases/download/%s/operadriver_%s.zip" % (latest,
860                                                                                                                self.platform_string())
861        unzip(get(url).raw, dest)
862
863        operadriver_dir = os.path.join(dest, "operadriver_%s" % self.platform_string())
864        shutil.move(os.path.join(operadriver_dir, "operadriver"), dest)
865        shutil.rmtree(operadriver_dir)
866
867        path = find_executable("operadriver")
868        st = os.stat(path)
869        os.chmod(path, st.st_mode | stat.S_IEXEC)
870        return path
871
872    def version(self, binary=None, webdriver_binary=None):
873        """Retrieve the release version of the installed browser."""
874        binary = binary or self.binary
875        try:
876            output = call(binary, "--version")
877        except subprocess.CalledProcessError:
878            self.logger.warning("Failed to call %s" % binary)
879            return None
880        m = re.search(r"[0-9\.]+( [a-z]+)?$", output.strip())
881        if m:
882            return m.group(0)
883
884
885class EdgeChromium(Browser):
886    """MicrosoftEdge-specific interface."""
887    platform = {
888        "Linux": "linux",
889        "Windows": "win",
890        "Darwin": "macos"
891    }.get(uname[0])
892    product = "edgechromium"
893    edgedriver_name = "msedgedriver"
894    requirements = "requirements_edge_chromium.txt"
895
896    def download(self, dest=None, channel=None, rename=None):
897        raise NotImplementedError
898
899    def install(self, dest=None, channel=None):
900        raise NotImplementedError
901
902    def find_binary(self, venv_path=None, channel=None):
903        binary = None
904        if self.platform == "win":
905            binaryname = "msedge"
906            binary = find_executable(binaryname)
907            if not binary:
908                # Use paths from different Edge channels starting with Release\Beta\Dev\Canary
909                winpaths = [os.path.expanduser("~\\AppData\\Local\\Microsoft\\Edge\\Application"),
910                            os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge\\Application"),
911                            os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge Beta\\Application"),
912                            os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Microsoft\\Edge Dev\\Application"),
913                            os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge\\Application"),
914                            os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge Beta\\Application"),
915                            os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Microsoft\\Edge Dev\\Application"),
916                            os.path.expanduser("~\\AppData\Local\\Microsoft\\Edge SxS\\Application")]
917                return find_executable(binaryname, os.pathsep.join(winpaths))
918        if self.platform == "macos":
919            binaryname = "Microsoft Edge Canary"
920            binary = find_executable(binaryname)
921            if not binary:
922                macpaths = ["/Applications/Microsoft Edge.app/Contents/MacOS",
923                            os.path.expanduser("~/Applications/Microsoft Edge.app/Contents/MacOS"),
924                            "/Applications/Microsoft Edge Canary.app/Contents/MacOS",
925                            os.path.expanduser("~/Applications/Microsoft Edge Canary.app/Contents/MacOS")]
926                return find_executable("Microsoft Edge Canary", os.pathsep.join(macpaths))
927        return binary
928
929    def find_webdriver(self, channel=None):
930        return find_executable("msedgedriver")
931
932    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
933        if self.platform != "win" and self.platform != "macos":
934            raise ValueError("Only Windows and Mac platforms are currently supported")
935
936        if dest is None:
937            dest = os.pwd
938
939        if channel is None:
940            version_url = "https://msedgedriver.azureedge.net/LATEST_DEV"
941        else:
942            version_url = "https://msedgedriver.azureedge.net/LATEST_%s" % channel.upper()
943        version = get(version_url).text.strip()
944
945        if self.platform == "macos":
946            bits = "mac64"
947            edgedriver_path = os.path.join(dest, self.edgedriver_name)
948        else:
949            bits = "win64" if uname[4] == "x86_64" else "win32"
950            edgedriver_path = os.path.join(dest, "%s.exe" % self.edgedriver_name)
951        url = "https://msedgedriver.azureedge.net/%s/edgedriver_%s.zip" % (version, bits)
952
953        # cleanup existing Edge driver files to avoid access_denied errors when unzipping
954        if os.path.isfile(edgedriver_path):
955            # remove read-only attribute
956            os.chmod(edgedriver_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777
957            print("Delete %s file" % edgedriver_path)
958            os.remove(edgedriver_path)
959        driver_notes_path = os.path.join(dest, "Driver_notes")
960        if os.path.isdir(driver_notes_path):
961            print("Delete %s folder" % driver_notes_path)
962            shutil.rmtree(driver_notes_path, ignore_errors=False, onerror=handle_remove_readonly)
963
964        self.logger.info("Downloading MSEdgeDriver from %s" % url)
965        unzip(get(url).raw, dest)
966        if os.path.isfile(edgedriver_path):
967            self.logger.info("Successfully downloaded MSEdgeDriver to %s" % edgedriver_path)
968        return find_executable(self.edgedriver_name, dest)
969
970    def version(self, binary=None, webdriver_binary=None):
971        if binary is None:
972            binary = self.find_binary()
973        if self.platform != "win":
974            try:
975                version_string = call(binary, "--version").strip()
976            except subprocess.CalledProcessError:
977                self.logger.warning("Failed to call %s" % binary)
978                return None
979            m = re.match(r"Microsoft Edge (.*) ", version_string)
980            if not m:
981                self.logger.warning("Failed to extract version from: %s" % version_string)
982                return None
983            return m.group(1)
984        else:
985            if binary is not None:
986                return _get_fileversion(binary, self.logger)
987            self.logger.warning("Failed to find Edge binary.")
988            return None
989
990
991class Edge(Browser):
992    """Edge-specific interface."""
993
994    product = "edge"
995    requirements = "requirements_edge.txt"
996
997    def download(self, dest=None, channel=None, rename=None):
998        raise NotImplementedError
999
1000    def install(self, dest=None, channel=None):
1001        raise NotImplementedError
1002
1003    def find_binary(self, venv_path=None, channel=None):
1004        raise NotImplementedError
1005
1006    def find_webdriver(self, channel=None):
1007        return find_executable("MicrosoftWebDriver")
1008
1009    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1010        raise NotImplementedError
1011
1012    def version(self, binary=None, webdriver_binary=None):
1013        command = "(Get-AppxPackage Microsoft.MicrosoftEdge).Version"
1014        try:
1015            return call("powershell.exe", command).strip()
1016        except (subprocess.CalledProcessError, OSError):
1017            self.logger.warning("Failed to call %s in PowerShell" % command)
1018            return None
1019
1020
1021class EdgeWebDriver(Edge):
1022    product = "edge_webdriver"
1023
1024
1025class InternetExplorer(Browser):
1026    """Internet Explorer-specific interface."""
1027
1028    product = "ie"
1029    requirements = "requirements_ie.txt"
1030
1031    def download(self, dest=None, channel=None, rename=None):
1032        raise NotImplementedError
1033
1034    def install(self, dest=None, channel=None):
1035        raise NotImplementedError
1036
1037    def find_binary(self, venv_path=None, channel=None):
1038        raise NotImplementedError
1039
1040    def find_webdriver(self, channel=None):
1041        return find_executable("IEDriverServer.exe")
1042
1043    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1044        raise NotImplementedError
1045
1046    def version(self, binary=None, webdriver_binary=None):
1047        return None
1048
1049
1050class Safari(Browser):
1051    """Safari-specific interface.
1052
1053    Includes installation, webdriver installation, and wptrunner setup methods.
1054    """
1055
1056    product = "safari"
1057    requirements = "requirements_safari.txt"
1058
1059    def download(self, dest=None, channel=None, rename=None):
1060        raise NotImplementedError
1061
1062    def install(self, dest=None, channel=None):
1063        raise NotImplementedError
1064
1065    def find_binary(self, venv_path=None, channel=None):
1066        raise NotImplementedError
1067
1068    def find_webdriver(self, channel=None):
1069        path = None
1070        if channel == "preview":
1071            path = "/Applications/Safari Technology Preview.app/Contents/MacOS"
1072        return find_executable("safaridriver", path)
1073
1074    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1075        raise NotImplementedError
1076
1077    def version(self, binary=None, webdriver_binary=None):
1078        if webdriver_binary is None:
1079            self.logger.warning("Cannot find Safari version without safaridriver")
1080            return None
1081        # Use `safaridriver --version` to get the version. Example output:
1082        # "Included with Safari 12.1 (14607.1.11)"
1083        # "Included with Safari Technology Preview (Release 67, 13607.1.9.0.1)"
1084        # The `--version` flag was added in STP 67, so allow the call to fail.
1085        try:
1086            version_string = call(webdriver_binary, "--version").strip()
1087        except subprocess.CalledProcessError:
1088            self.logger.warning("Failed to call %s --version" % webdriver_binary)
1089            return None
1090        m = re.match(br"Included with Safari (.*)", version_string)
1091        if not m:
1092            self.logger.warning("Failed to extract version from: %s" % version_string)
1093            return None
1094        return m.group(1).decode()
1095
1096
1097class Servo(Browser):
1098    """Servo-specific interface."""
1099
1100    product = "servo"
1101    requirements = "requirements_servo.txt"
1102
1103    def platform_components(self):
1104        platform = {
1105            "Linux": "linux",
1106            "Windows": "win",
1107            "Darwin": "mac"
1108        }.get(uname[0])
1109
1110        if platform is None:
1111            raise ValueError("Unable to construct a valid Servo package for current platform")
1112
1113        if platform == "linux":
1114            extension = ".tar.gz"
1115            decompress = untar
1116        elif platform == "win" or platform == "mac":
1117            raise ValueError("Unable to construct a valid Servo package for current platform")
1118
1119        return (platform, extension, decompress)
1120
1121    def _get(self, channel="nightly"):
1122        if channel != "nightly":
1123            raise ValueError("Only nightly versions of Servo are available")
1124
1125        platform, extension, _ = self.platform_components()
1126        url = "https://download.servo.org/nightly/%s/servo-latest%s" % (platform, extension)
1127        return get(url)
1128
1129    def download(self, dest=None, channel="nightly", rename=None):
1130        if dest is None:
1131            dest = os.pwd
1132
1133        resp = self._get(dest, channel)
1134        _, extension, _ = self.platform_components()
1135
1136        filename = rename if rename is not None else "servo-latest"
1137        with open(os.path.join(dest, "%s%s" % (filename, extension,)), "w") as f:
1138            f.write(resp.content)
1139
1140    def install(self, dest=None, channel="nightly"):
1141        """Install latest Browser Engine."""
1142        if dest is None:
1143            dest = os.pwd
1144
1145        _, _, decompress = self.platform_components()
1146
1147        resp = self._get(channel)
1148        decompress(resp.raw, dest=dest)
1149        path = find_executable("servo", os.path.join(dest, "servo"))
1150        st = os.stat(path)
1151        os.chmod(path, st.st_mode | stat.S_IEXEC)
1152        return path
1153
1154    def find_binary(self, venv_path=None, channel=None):
1155        path = find_executable("servo", os.path.join(venv_path, "servo"))
1156        if path is None:
1157            path = find_executable("servo")
1158        return path
1159
1160    def find_webdriver(self, channel=None):
1161        return None
1162
1163    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1164        raise NotImplementedError
1165
1166    def version(self, binary=None, webdriver_binary=None):
1167        """Retrieve the release version of the installed browser."""
1168        output = call(binary, "--version")
1169        m = re.search(r"Servo ([0-9\.]+-[a-f0-9]+)?(-dirty)?$", output.strip())
1170        if m:
1171            return m.group(0)
1172
1173
1174class ServoWebDriver(Servo):
1175    product = "servodriver"
1176
1177
1178class Sauce(Browser):
1179    """Sauce-specific interface."""
1180
1181    product = "sauce"
1182    requirements = "requirements_sauce.txt"
1183
1184    def download(self, dest=None, channel=None, rename=None):
1185        raise NotImplementedError
1186
1187    def install(self, dest=None, channel=None):
1188        raise NotImplementedError
1189
1190    def find_binary(self, venev_path=None, channel=None):
1191        raise NotImplementedError
1192
1193    def find_webdriver(self, channel=None):
1194        raise NotImplementedError
1195
1196    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1197        raise NotImplementedError
1198
1199    def version(self, binary=None, webdriver_binary=None):
1200        return None
1201
1202
1203class WebKit(Browser):
1204    """WebKit-specific interface."""
1205
1206    product = "webkit"
1207    requirements = "requirements_webkit.txt"
1208
1209    def download(self, dest=None, channel=None, rename=None):
1210        raise NotImplementedError
1211
1212    def install(self, dest=None, channel=None):
1213        raise NotImplementedError
1214
1215    def find_binary(self, venv_path=None, channel=None):
1216        return None
1217
1218    def find_webdriver(self, channel=None):
1219        return None
1220
1221    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1222        raise NotImplementedError
1223
1224    def version(self, binary=None, webdriver_binary=None):
1225        return None
1226
1227
1228class WebKitGTKMiniBrowser(WebKit):
1229
1230    def find_binary(self, venv_path=None, channel=None):
1231        libexecpaths = ["/usr/libexec/webkit2gtk-4.0"]  # Fedora path
1232        triplet = "x86_64-linux-gnu"
1233        # Try to use GCC to detect this machine triplet
1234        gcc = find_executable("gcc")
1235        if gcc:
1236            try:
1237                triplet = call(gcc, "-dumpmachine").decode().strip()
1238            except subprocess.CalledProcessError:
1239                pass
1240        # Add Debian/Ubuntu path
1241        libexecpaths.append("/usr/lib/%s/webkit2gtk-4.0" % triplet)
1242        if channel == "nightly":
1243            libexecpaths.append("/opt/webkitgtk/nightly")
1244        return find_executable("MiniBrowser", os.pathsep.join(libexecpaths))
1245
1246    def find_webdriver(self, channel=None):
1247        path = os.environ['PATH']
1248        if channel == "nightly":
1249            path = "%s:%s" % (path, "/opt/webkitgtk/nightly")
1250        return find_executable("WebKitWebDriver", path)
1251
1252    def version(self, binary=None, webdriver_binary=None):
1253        if binary is None:
1254            return None
1255        try:  # WebKitGTK MiniBrowser before 2.26.0 doesn't support --version
1256            output = call(binary, "--version").decode().strip()
1257        except subprocess.CalledProcessError:
1258            return None
1259        # Example output: "WebKitGTK 2.26.1"
1260        if output:
1261            m = re.match(r"WebKitGTK (.+)", output)
1262            if not m:
1263                self.logger.warning("Failed to extract version from: %s" % output)
1264                return None
1265            return m.group(1)
1266        return None
1267
1268
1269class Epiphany(Browser):
1270    """Epiphany-specific interface."""
1271
1272    product = "epiphany"
1273    requirements = "requirements_epiphany.txt"
1274
1275    def download(self, dest=None, channel=None, rename=None):
1276        raise NotImplementedError
1277
1278    def install(self, dest=None, channel=None):
1279        raise NotImplementedError
1280
1281    def find_binary(self, venv_path=None, channel=None):
1282        return find_executable("epiphany")
1283
1284    def find_webdriver(self, channel=None):
1285        return find_executable("WebKitWebDriver")
1286
1287    def install_webdriver(self, dest=None, channel=None, browser_binary=None):
1288        raise NotImplementedError
1289
1290    def version(self, binary=None, webdriver_binary=None):
1291        if binary is None:
1292            return None
1293        output = call(binary, "--version")
1294        if output:
1295            # Stable release output looks like: "Web 3.30.2"
1296            # Tech Preview output looks like "Web 3.31.3-88-g97db4f40f"
1297            return output.split()[1]
1298        return None
1299