1import os
2import subprocess
3import sys
4from collections import defaultdict
5from typing import Any, ClassVar, Dict, Type
6from urllib.parse import urljoin
7
8from .wptmanifest.parser import atoms
9
10atom_reset = atoms["Reset"]
11enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest"}
12
13
14class Result(object):
15    def __init__(self,
16                 status,
17                 message,
18                 expected=None,
19                 extra=None,
20                 stack=None,
21                 known_intermittent=None):
22        if status not in self.statuses:
23            raise ValueError("Unrecognised status %s" % status)
24        self.status = status
25        self.message = message
26        self.expected = expected
27        self.known_intermittent = known_intermittent if known_intermittent is not None else []
28        self.extra = extra if extra is not None else {}
29        self.stack = stack
30
31    def __repr__(self):
32        return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
33
34
35class SubtestResult(object):
36    def __init__(self, name, status, message, stack=None, expected=None, known_intermittent=None):
37        self.name = name
38        if status not in self.statuses:
39            raise ValueError("Unrecognised status %s" % status)
40        self.status = status
41        self.message = message
42        self.stack = stack
43        self.expected = expected
44        self.known_intermittent = known_intermittent if known_intermittent is not None else []
45
46    def __repr__(self):
47        return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
48
49
50class TestharnessResult(Result):
51    default_expected = "OK"
52    statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH", "PRECONDITION_FAILED"}
53
54
55class TestharnessSubtestResult(SubtestResult):
56    default_expected = "PASS"
57    statuses = {"PASS", "FAIL", "TIMEOUT", "NOTRUN", "PRECONDITION_FAILED"}
58
59
60class ReftestResult(Result):
61    default_expected = "PASS"
62    statuses = {"PASS", "FAIL", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT",
63                "CRASH"}
64
65
66class WdspecResult(Result):
67    default_expected = "OK"
68    statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"}
69
70
71class WdspecSubtestResult(SubtestResult):
72    default_expected = "PASS"
73    statuses = {"PASS", "FAIL", "ERROR"}
74
75
76class CrashtestResult(Result):
77    default_expected = "PASS"
78    statuses = {"PASS", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT",
79                "CRASH"}
80
81
82def get_run_info(metadata_root, product, **kwargs):
83    return RunInfo(metadata_root, product, **kwargs)
84
85
86class RunInfo(Dict[str, Any]):
87    def __init__(self, metadata_root, product, debug,
88                 browser_version=None,
89                 browser_channel=None,
90                 verify=None,
91                 extras=None,
92                 enable_webrender=False):
93        import mozinfo
94        self._update_mozinfo(metadata_root)
95        self.update(mozinfo.info)
96
97        from .update.tree import GitTree
98        try:
99            # GitTree.__init__ throws if we are not in a git tree.
100            rev = GitTree(log_error=False).rev
101        except (OSError, subprocess.CalledProcessError):
102            rev = None
103        if rev:
104            self["revision"] = rev.decode("utf-8")
105
106        self["python_version"] = sys.version_info.major
107        self["product"] = product
108        if debug is not None:
109            self["debug"] = debug
110        elif "debug" not in self:
111            # Default to release
112            self["debug"] = False
113        if browser_version:
114            self["browser_version"] = browser_version
115        if browser_channel:
116            self["browser_channel"] = browser_channel
117
118        self["verify"] = verify
119        if "wasm" not in self:
120            self["wasm"] = False
121        if extras is not None:
122            self.update(extras)
123
124        self["headless"] = extras.get("headless", False)
125        self["webrender"] = enable_webrender
126
127    def _update_mozinfo(self, metadata_root):
128        """Add extra build information from a mozinfo.json file in a parent
129        directory"""
130        import mozinfo
131
132        path = metadata_root
133        dirs = set()
134        while path != os.path.expanduser('~'):
135            if path in dirs:
136                break
137            dirs.add(str(path))
138            path = os.path.dirname(path)
139
140        mozinfo.find_and_update_from_json(*dirs)
141
142
143def server_protocol(manifest_item):
144    if hasattr(manifest_item, "h2") and manifest_item.h2:
145        return "h2"
146    if hasattr(manifest_item, "https") and manifest_item.https:
147        return "https"
148    return "http"
149
150
151class Test(object):
152
153    result_cls = None  # type: ClassVar[Type[Result]]
154    subtest_result_cls = None  # type: ClassVar[Type[SubtestResult]]
155    test_type = None  # type: ClassVar[str]
156
157    default_timeout = 10  # seconds
158    long_timeout = 60  # seconds
159
160    def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata,
161                 timeout=None, path=None, protocol="http", subdomain=False,
162                 quic=False):
163        self.url_base = url_base
164        self.tests_root = tests_root
165        self.url = url
166        self._inherit_metadata = inherit_metadata
167        self._test_metadata = test_metadata
168        self.timeout = timeout if timeout is not None else self.default_timeout
169        self.path = path
170
171        self.subdomain = subdomain
172        self.environment = {"url_base": url_base,
173                            "protocol": protocol,
174                            "prefs": self.prefs,
175                            "quic": quic}
176
177    def __eq__(self, other):
178        if not isinstance(other, Test):
179            return False
180        return self.id == other.id
181
182    # Python 2 does not have this delegation, while Python 3 does.
183    def __ne__(self, other):
184        return not self.__eq__(other)
185
186    def update_metadata(self, metadata=None):
187        if metadata is None:
188            metadata = {}
189        return metadata
190
191    @classmethod
192    def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_metadata):
193        timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout
194        return cls(manifest_file.url_base,
195                   manifest_file.tests_root,
196                   manifest_item.url,
197                   inherit_metadata,
198                   test_metadata,
199                   timeout=timeout,
200                   path=os.path.join(manifest_file.tests_root, manifest_item.path),
201                   protocol=server_protocol(manifest_item),
202                   subdomain=manifest_item.subdomain)
203
204    @property
205    def id(self):
206        return self.url
207
208    @property
209    def keys(self):
210        return tuple()
211
212    @property
213    def abs_path(self):
214        return os.path.join(self.tests_root, self.path)
215
216    def _get_metadata(self, subtest=None):
217        if self._test_metadata is not None and subtest is not None:
218            return self._test_metadata.get_subtest(subtest)
219        else:
220            return self._test_metadata
221
222    def itermeta(self, subtest=None):
223        if self._test_metadata is not None:
224            if subtest is not None:
225                subtest_meta = self._get_metadata(subtest)
226                if subtest_meta is not None:
227                    yield subtest_meta
228            yield self._get_metadata()
229        for metadata in reversed(self._inherit_metadata):
230            yield metadata
231
232    def disabled(self, subtest=None):
233        for meta in self.itermeta(subtest):
234            disabled = meta.disabled
235            if disabled is not None:
236                return disabled
237        return None
238
239    @property
240    def restart_after(self):
241        for meta in self.itermeta(None):
242            restart_after = meta.restart_after
243            if restart_after is not None:
244                return True
245        return False
246
247    @property
248    def leaks(self):
249        for meta in self.itermeta(None):
250            leaks = meta.leaks
251            if leaks is not None:
252                return leaks
253        return False
254
255    @property
256    def min_assertion_count(self):
257        for meta in self.itermeta(None):
258            count = meta.min_assertion_count
259            if count is not None:
260                return count
261        return 0
262
263    @property
264    def max_assertion_count(self):
265        for meta in self.itermeta(None):
266            count = meta.max_assertion_count
267            if count is not None:
268                return count
269        return 0
270
271    @property
272    def lsan_disabled(self):
273        for meta in self.itermeta():
274            if meta.lsan_disabled is not None:
275                return meta.lsan_disabled
276        return False
277
278    @property
279    def lsan_allowed(self):
280        lsan_allowed = set()
281        for meta in self.itermeta():
282            lsan_allowed |= meta.lsan_allowed
283            if atom_reset in lsan_allowed:
284                lsan_allowed.remove(atom_reset)
285                break
286        return lsan_allowed
287
288    @property
289    def lsan_max_stack_depth(self):
290        for meta in self.itermeta(None):
291            depth = meta.lsan_max_stack_depth
292            if depth is not None:
293                return depth
294        return None
295
296    @property
297    def mozleak_allowed(self):
298        mozleak_allowed = set()
299        for meta in self.itermeta():
300            mozleak_allowed |= meta.leak_allowed
301            if atom_reset in mozleak_allowed:
302                mozleak_allowed.remove(atom_reset)
303                break
304        return mozleak_allowed
305
306    @property
307    def mozleak_threshold(self):
308        rv = {}
309        for meta in self.itermeta(None):
310            threshold = meta.leak_threshold
311            for key, value in threshold.items():
312                if key not in rv:
313                    rv[key] = value
314        return rv
315
316    @property
317    def tags(self):
318        tags = set()
319        for meta in self.itermeta():
320            meta_tags = meta.tags
321            tags |= meta_tags
322            if atom_reset in meta_tags:
323                tags.remove(atom_reset)
324                break
325
326        tags.add("dir:%s" % self.id.lstrip("/").split("/")[0])
327
328        return tags
329
330    @property
331    def prefs(self):
332        prefs = {}
333        for meta in reversed(list(self.itermeta())):
334            meta_prefs = meta.prefs
335            if atom_reset in meta_prefs:
336                del meta_prefs[atom_reset]
337                prefs = {}
338            prefs.update(meta_prefs)
339        return prefs
340
341    def expected(self, subtest=None):
342        if subtest is None:
343            default = self.result_cls.default_expected
344        else:
345            default = self.subtest_result_cls.default_expected
346
347        metadata = self._get_metadata(subtest)
348        if metadata is None:
349            return default
350
351        try:
352            expected = metadata.get("expected")
353            if isinstance(expected, str):
354                return expected
355            elif isinstance(expected, list):
356                return expected[0]
357            elif expected is None:
358                return default
359        except KeyError:
360            return default
361
362    def implementation_status(self):
363        implementation_status = None
364        for meta in self.itermeta():
365            implementation_status = meta.implementation_status
366            if implementation_status:
367                return implementation_status
368
369        # assuming no specific case, we are implementing it
370        return "implementing"
371
372    def known_intermittent(self, subtest=None):
373        metadata = self._get_metadata(subtest)
374        if metadata is None:
375            return []
376
377        try:
378            expected = metadata.get("expected")
379            if isinstance(expected, list):
380                return expected[1:]
381            return []
382        except KeyError:
383            return []
384
385    def expect_any_subtest_status(self):
386        metadata = self._get_metadata()
387        if metadata is None:
388            return False
389        try:
390            # This key is used by the Blink CI to ignore subtest statuses
391            metadata.get("blink_expect_any_subtest_status")
392            return True
393        except KeyError:
394            return False
395
396    def __repr__(self):
397        return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
398
399
400class TestharnessTest(Test):
401    result_cls = TestharnessResult
402    subtest_result_cls = TestharnessSubtestResult
403    test_type = "testharness"
404
405    def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata,
406                 timeout=None, path=None, protocol="http", testdriver=False,
407                 jsshell=False, scripts=None, subdomain=False, quic=False):
408        Test.__init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, timeout,
409                      path, protocol, subdomain, quic)
410
411        self.testdriver = testdriver
412        self.jsshell = jsshell
413        self.scripts = scripts or []
414
415    @classmethod
416    def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_metadata):
417        timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout
418        testdriver = manifest_item.testdriver if hasattr(manifest_item, "testdriver") else False
419        jsshell = manifest_item.jsshell if hasattr(manifest_item, "jsshell") else False
420        quic = manifest_item.quic if hasattr(manifest_item, "quic") else False
421        script_metadata = manifest_item.script_metadata or []
422        scripts = [v for (k, v) in script_metadata
423                   if k == "script"]
424        return cls(manifest_file.url_base,
425                   manifest_file.tests_root,
426                   manifest_item.url,
427                   inherit_metadata,
428                   test_metadata,
429                   timeout=timeout,
430                   path=os.path.join(manifest_file.tests_root, manifest_item.path),
431                   protocol=server_protocol(manifest_item),
432                   testdriver=testdriver,
433                   jsshell=jsshell,
434                   scripts=scripts,
435                   subdomain=manifest_item.subdomain,
436                   quic=quic)
437
438    @property
439    def id(self):
440        return self.url
441
442
443class ManualTest(Test):
444    test_type = "manual"
445
446    @property
447    def id(self):
448        return self.url
449
450
451class ReftestTest(Test):
452    """A reftest
453
454    A reftest should be considered to pass if one of its references matches (see below) *and* the
455    reference passes if it has any references recursively.
456
457    Attributes:
458        references (List[Tuple[str, str]]): a list of alternate references, where one must match for the test to pass
459        viewport_size (Optional[Tuple[int, int]]): size of the viewport for this test, if not default
460        dpi (Optional[int]): dpi to use when rendering this test, if not default
461
462    """
463    result_cls = ReftestResult
464    test_type = "reftest"
465
466    def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, references,
467                 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None,
468                 protocol="http", subdomain=False, quic=False):
469        Test.__init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, timeout,
470                      path, protocol, subdomain, quic)
471
472        for _, ref_type in references:
473            if ref_type not in ("==", "!="):
474                raise ValueError
475
476        self.references = references
477        self.viewport_size = self.get_viewport_size(viewport_size)
478        self.dpi = dpi
479        self._fuzzy = fuzzy or {}
480
481    @classmethod
482    def cls_kwargs(cls, manifest_test):
483        return {"viewport_size": manifest_test.viewport_size,
484                "dpi": manifest_test.dpi,
485                "protocol": server_protocol(manifest_test),
486                "fuzzy": manifest_test.fuzzy}
487
488    @classmethod
489    def from_manifest(cls,
490                      manifest_file,
491                      manifest_test,
492                      inherit_metadata,
493                      test_metadata):
494
495        timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout
496        quic = manifest_test.quic if hasattr(manifest_test, "quic") else False
497
498        url = manifest_test.url
499
500        node = cls(manifest_file.url_base,
501                   manifest_file.tests_root,
502                   manifest_test.url,
503                   inherit_metadata,
504                   test_metadata,
505                   [],
506                   timeout=timeout,
507                   path=manifest_test.path,
508                   subdomain=manifest_test.subdomain,
509                   quic=quic,
510                   **cls.cls_kwargs(manifest_test))
511
512        refs_by_type = defaultdict(list)
513
514        for ref_url, ref_type in manifest_test.references:
515            refs_by_type[ref_type].append(ref_url)
516
517        # Construct a list of all the mismatches, where we end up with mismatch_1 != url !=
518        # mismatch_2 != url != mismatch_3 etc.
519        #
520        # Per the logic documented above, this means that none of the mismatches provided match,
521        mismatch_walk = None
522        if refs_by_type["!="]:
523            mismatch_walk = ReftestTest(manifest_file.url_base,
524                                        manifest_file.tests_root,
525                                        refs_by_type["!="][0],
526                                        [],
527                                        None,
528                                        [])
529            cmp_ref = mismatch_walk
530            for ref_url in refs_by_type["!="][1:]:
531                cmp_self = ReftestTest(manifest_file.url_base,
532                                       manifest_file.tests_root,
533                                       url,
534                                       [],
535                                       None,
536                                       [])
537                cmp_ref.references.append((cmp_self, "!="))
538                cmp_ref = ReftestTest(manifest_file.url_base,
539                                      manifest_file.tests_root,
540                                      ref_url,
541                                      [],
542                                      None,
543                                      [])
544                cmp_self.references.append((cmp_ref, "!="))
545
546        if mismatch_walk is None:
547            mismatch_refs = []
548        else:
549            mismatch_refs = [(mismatch_walk, "!=")]
550
551        if refs_by_type["=="]:
552            # For each == ref, add a reference to this node whose tail is the mismatch list.
553            # Per the logic documented above, this means any one of the matches must pass plus all the mismatches.
554            for ref_url in refs_by_type["=="]:
555                ref = ReftestTest(manifest_file.url_base,
556                                  manifest_file.tests_root,
557                                  ref_url,
558                                  [],
559                                  None,
560                                  mismatch_refs)
561                node.references.append((ref, "=="))
562        else:
563            # Otherwise, we just add the mismatches directly as we are immediately into the
564            # mismatch chain with no alternates.
565            node.references.extend(mismatch_refs)
566
567        return node
568
569    def update_metadata(self, metadata):
570        if "url_count" not in metadata:
571            metadata["url_count"] = defaultdict(int)
572        for reference, _ in self.references:
573            # We assume a naive implementation in which a url with multiple
574            # possible screenshots will need to take both the lhs and rhs screenshots
575            # for each possible match
576            metadata["url_count"][(self.environment["protocol"], reference.url)] += 1
577            reference.update_metadata(metadata)
578        return metadata
579
580    def get_viewport_size(self, override):
581        return override
582
583    @property
584    def id(self):
585        return self.url
586
587    @property
588    def keys(self):
589        return ("reftype", "refurl")
590
591    @property
592    def fuzzy(self):
593        return self._fuzzy
594
595    @property
596    def fuzzy_override(self):
597        values = {}
598        for meta in reversed(list(self.itermeta(None))):
599            value = meta.fuzzy
600            if not value:
601                continue
602            if atom_reset in value:
603                value.remove(atom_reset)
604                values = {}
605            for key, data in value:
606                if isinstance(key, (tuple, list)):
607                    key = list(key)
608                    key[0] = urljoin(self.url, key[0])
609                    key[1] = urljoin(self.url, key[1])
610                    key = tuple(key)
611                elif key:
612                    # Key is just a relative url to a ref
613                    key = urljoin(self.url, key)
614                values[key] = data
615        return values
616
617    @property
618    def page_ranges(self):
619        return {}
620
621
622class PrintReftestTest(ReftestTest):
623    test_type = "print-reftest"
624
625    def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, references,
626                 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None,
627                 page_ranges=None, protocol="http", subdomain=False, quic=False):
628        super(PrintReftestTest, self).__init__(url_base, tests_root, url, inherit_metadata, test_metadata,
629                                               references, timeout, path, viewport_size, dpi,
630                                               fuzzy, protocol, subdomain=subdomain, quic=quic)
631        self._page_ranges = page_ranges
632
633    @classmethod
634    def cls_kwargs(cls, manifest_test):
635        rv = super(PrintReftestTest, cls).cls_kwargs(manifest_test)
636        rv["page_ranges"] = manifest_test.page_ranges
637        return rv
638
639    def get_viewport_size(self, override):
640        assert override is None
641        return (5*2.54, 3*2.54)
642
643    @property
644    def page_ranges(self):
645        return self._page_ranges
646
647
648class WdspecTest(Test):
649    result_cls = WdspecResult
650    subtest_result_cls = WdspecSubtestResult
651    test_type = "wdspec"
652
653    default_timeout = 25
654    long_timeout = 180  # 3 minutes
655
656
657class CrashTest(Test):
658    result_cls = CrashtestResult
659    test_type = "crashtest"
660
661
662manifest_test_cls = {"reftest": ReftestTest,
663                     "print-reftest": PrintReftestTest,
664                     "testharness": TestharnessTest,
665                     "manual": ManualTest,
666                     "wdspec": WdspecTest,
667                     "crashtest": CrashTest}
668
669
670def from_manifest(manifest_file, manifest_test, inherit_metadata, test_metadata):
671    test_cls = manifest_test_cls[manifest_test.item_type]
672    return test_cls.from_manifest(manifest_file, manifest_test, inherit_metadata, test_metadata)
673