1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
6import os
7import re
8import sys
9from collections import defaultdict
10
11import mozpack.path as mozpath
12from moztest.resolve import TestResolver
13
14from ..cli import BaseTryParser
15from ..push import build, push_to_try
16
17here = os.path.abspath(os.path.dirname(__file__))
18
19
20class SyntaxParser(BaseTryParser):
21    name = "syntax"
22    arguments = [
23        [
24            ["paths"],
25            {
26                "nargs": "*",
27                "default": [],
28                "help": "Paths to search for tests to run on try.",
29            },
30        ],
31        [
32            ["-b", "--build"],
33            {
34                "dest": "builds",
35                "default": "do",
36                "help": "Build types to run (d for debug, o for optimized).",
37            },
38        ],
39        [
40            ["-p", "--platform"],
41            {
42                "dest": "platforms",
43                "action": "append",
44                "help": "Platforms to run (required if not found in the environment as "
45                "AUTOTRY_PLATFORM_HINT).",
46            },
47        ],
48        [
49            ["-u", "--unittests"],
50            {
51                "dest": "tests",
52                "action": "append",
53                "help": "Test suites to run in their entirety.",
54            },
55        ],
56        [
57            ["-t", "--talos"],
58            {
59                "action": "append",
60                "help": "Talos suites to run.",
61            },
62        ],
63        [
64            ["-j", "--jobs"],
65            {
66                "action": "append",
67                "help": "Job tasks to run.",
68            },
69        ],
70        [
71            ["--tag"],
72            {
73                "dest": "tags",
74                "action": "append",
75                "help": "Restrict tests to the given tag (may be specified multiple times).",
76            },
77        ],
78        [
79            ["--and"],
80            {
81                "action": "store_true",
82                "dest": "intersection",
83                "help": "When -u and paths are supplied run only the intersection of the "
84                "tests specified by the two arguments.",
85            },
86        ],
87        [
88            ["--no-artifact"],
89            {
90                "action": "store_true",
91                "help": "Disable artifact builds even if --enable-artifact-builds is set "
92                "in the mozconfig.",
93            },
94        ],
95        [
96            ["-v", "--verbose"],
97            {
98                "dest": "verbose",
99                "action": "store_true",
100                "default": False,
101                "help": "Print detailed information about the resulting test selection "
102                "and commands performed.",
103            },
104        ],
105    ]
106
107    # Arguments we will accept on the command line and pass through to try
108    # syntax with no further intervention. The set is taken from
109    # http://trychooser.pub.build.mozilla.org with a few additions.
110    #
111    # Note that the meaning of store_false and store_true arguments is
112    # not preserved here, as we're only using these to echo the literal
113    # arguments to another consumer. Specifying either store_false or
114    # store_true here will have an equivalent effect.
115    pass_through_arguments = {
116        "--rebuild": {
117            "action": "store",
118            "dest": "rebuild",
119            "help": "Re-trigger all test jobs (up to 20 times)",
120        },
121        "--rebuild-talos": {
122            "action": "store",
123            "dest": "rebuild_talos",
124            "help": "Re-trigger all talos jobs",
125        },
126        "--interactive": {
127            "action": "store_true",
128            "dest": "interactive",
129            "help": "Allow ssh-like access to running test containers",
130        },
131        "--no-retry": {
132            "action": "store_true",
133            "dest": "no_retry",
134            "help": "Do not retrigger failed tests",
135        },
136        "--setenv": {
137            "action": "append",
138            "dest": "setenv",
139            "help": "Set the corresponding variable in the test environment for "
140            "applicable harnesses.",
141        },
142        "-f": {
143            "action": "store_true",
144            "dest": "failure_emails",
145            "help": "Request failure emails only",
146        },
147        "--failure-emails": {
148            "action": "store_true",
149            "dest": "failure_emails",
150            "help": "Request failure emails only",
151        },
152        "-e": {
153            "action": "store_true",
154            "dest": "all_emails",
155            "help": "Request all emails",
156        },
157        "--all-emails": {
158            "action": "store_true",
159            "dest": "all_emails",
160            "help": "Request all emails",
161        },
162        "--artifact": {
163            "action": "store_true",
164            "dest": "artifact",
165            "help": "Force artifact builds where possible.",
166        },
167        "--upload-xdbs": {
168            "action": "store_true",
169            "dest": "upload_xdbs",
170            "help": "Upload XDB compilation db files generated by hazard build",
171        },
172    }
173    task_configs = []
174
175    def __init__(self, *args, **kwargs):
176        BaseTryParser.__init__(self, *args, **kwargs)
177
178        group = self.add_argument_group("pass-through arguments")
179        for arg, opts in self.pass_through_arguments.items():
180            group.add_argument(arg, **opts)
181
182
183class TryArgumentTokenizer:
184    symbols = [
185        ("separator", ","),
186        ("list_start", r"\["),
187        ("list_end", r"\]"),
188        ("item", r"([^,\[\]\s][^,\[\]]+)"),
189        ("space", r"\s+"),
190    ]
191    token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
192
193    def tokenize(self, data):
194        for match in self.token_re.finditer(data):
195            symbol = match.lastgroup
196            data = match.group(symbol)
197            if symbol == "space":
198                pass
199            else:
200                yield symbol, data
201
202
203class TryArgumentParser:
204    """Simple three-state parser for handling expressions
205    of the from "foo[sub item, another], bar,baz". This takes
206    input from the TryArgumentTokenizer and runs through a small
207    state machine, returning a dictionary of {top-level-item:[sub_items]}
208    i.e. the above would result in
209    {"foo":["sub item", "another"], "bar": [], "baz": []}
210    In the case of invalid input a ValueError is raised."""
211
212    EOF = object()
213
214    def __init__(self):
215        self.reset()
216
217    def reset(self):
218        self.tokens = None
219        self.current_item = None
220        self.data = {}
221        self.token = None
222        self.state = None
223
224    def parse(self, tokens):
225        self.reset()
226        self.tokens = tokens
227        self.consume()
228        self.state = self.item_state
229        while self.token[0] != self.EOF:
230            self.state()
231        return self.data
232
233    def consume(self):
234        try:
235            self.token = next(self.tokens)
236        except StopIteration:
237            self.token = (self.EOF, None)
238
239    def expect(self, *types):
240        if self.token[0] not in types:
241            raise ValueError(
242                "Error parsing try string, unexpected %s" % (self.token[0])
243            )
244
245    def item_state(self):
246        self.expect("item")
247        value = self.token[1].strip()
248        if value not in self.data:
249            self.data[value] = []
250        self.current_item = value
251        self.consume()
252        if self.token[0] == "separator":
253            self.consume()
254        elif self.token[0] == "list_start":
255            self.consume()
256            self.state = self.subitem_state
257        elif self.token[0] == self.EOF:
258            pass
259        else:
260            raise ValueError
261
262    def subitem_state(self):
263        self.expect("item")
264        value = self.token[1].strip()
265        self.data[self.current_item].append(value)
266        self.consume()
267        if self.token[0] == "separator":
268            self.consume()
269        elif self.token[0] == "list_end":
270            self.consume()
271            self.state = self.after_list_end_state
272        else:
273            raise ValueError
274
275    def after_list_end_state(self):
276        self.expect("separator")
277        self.consume()
278        self.state = self.item_state
279
280
281def parse_arg(arg):
282    tokenizer = TryArgumentTokenizer()
283    parser = TryArgumentParser()
284    return parser.parse(tokenizer.tokenize(arg))
285
286
287class AutoTry:
288
289    # Maps from flavors to the job names needed to run that flavour
290    flavor_jobs = {
291        "mochitest": ["mochitest-1", "mochitest-e10s-1"],
292        "xpcshell": ["xpcshell"],
293        "chrome": ["mochitest-o"],
294        "browser-chrome": [
295            "mochitest-browser-chrome-1",
296            "mochitest-e10s-browser-chrome-1",
297            "mochitest-browser-chrome-e10s-1",
298        ],
299        "devtools-chrome": [
300            "mochitest-devtools-chrome-1",
301            "mochitest-e10s-devtools-chrome-1",
302            "mochitest-devtools-chrome-e10s-1",
303        ],
304        "crashtest": ["crashtest", "crashtest-e10s"],
305        "reftest": ["reftest", "reftest-e10s"],
306        "remote": ["mochitest-remote"],
307        "web-platform-tests": ["web-platform-tests-1"],
308    }
309
310    flavor_suites = {
311        "mochitest": "mochitests",
312        "xpcshell": "xpcshell",
313        "chrome": "mochitest-o",
314        "browser-chrome": "mochitest-bc",
315        "devtools-chrome": "mochitest-dt",
316        "crashtest": "crashtest",
317        "reftest": "reftest",
318        "web-platform-tests": "web-platform-tests",
319    }
320
321    compiled_suites = [
322        "cppunit",
323        "gtest",
324        "jittest",
325    ]
326
327    common_suites = [
328        "cppunit",
329        "crashtest",
330        "firefox-ui-functional",
331        "geckoview",
332        "geckoview-junit",
333        "gtest",
334        "jittest",
335        "jsreftest",
336        "marionette",
337        "marionette-e10s",
338        "mochitests",
339        "reftest",
340        "robocop",
341        "web-platform-tests",
342        "xpcshell",
343    ]
344
345    def __init__(self):
346        self.topsrcdir = build.topsrcdir
347        self._resolver = None
348
349    @property
350    def resolver(self):
351        if self._resolver is None:
352            self._resolver = TestResolver.from_environment(cwd=here)
353        return self._resolver
354
355    @classmethod
356    def split_try_string(cls, data):
357        return re.findall(r"(?:\[.*?\]|\S)+", data)
358
359    def paths_by_flavor(self, paths=None, tags=None):
360        paths_by_flavor = defaultdict(set)
361
362        if not (paths or tags):
363            return dict(paths_by_flavor)
364
365        tests = list(self.resolver.resolve_tests(paths=paths, tags=tags))
366
367        for t in tests:
368            if t["flavor"] in self.flavor_suites:
369                flavor = t["flavor"]
370                if "subsuite" in t and t["subsuite"] == "devtools":
371                    flavor = "devtools-chrome"
372
373                if flavor in ["crashtest", "reftest"]:
374                    manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir)
375                    paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
376                elif "dir_relpath" in t:
377                    paths_by_flavor[flavor].add(t["dir_relpath"])
378                else:
379                    file_relpath = os.path.relpath(t["path"], self.topsrcdir)
380                    dir_relpath = os.path.dirname(file_relpath)
381                    paths_by_flavor[flavor].add(dir_relpath)
382
383        for flavor, path_set in paths_by_flavor.items():
384            paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
385
386        return dict(paths_by_flavor)
387
388    def deduplicate_prefixes(self, path_set, input_paths):
389        # Removes paths redundant to test selection in the given path set.
390        # If a path was passed on the commandline that is the prefix of a
391        # path in our set, we only need to include the specified prefix to
392        # run the intended tests (every test in "layout/base" will run if
393        # "layout" is passed to the reftest harness).
394        removals = set()
395        additions = set()
396
397        for path in path_set:
398            full_path = path
399            while path:
400                path, _ = os.path.split(path)
401                if path in input_paths:
402                    removals.add(full_path)
403                    additions.add(path)
404
405        return additions | (path_set - removals)
406
407    def remove_duplicates(self, paths_by_flavor, tests):
408        rv = {}
409        for item in paths_by_flavor:
410            if self.flavor_suites[item] not in tests:
411                rv[item] = paths_by_flavor[item].copy()
412        return rv
413
414    def calc_try_syntax(
415        self,
416        platforms,
417        tests,
418        talos,
419        jobs,
420        builds,
421        paths_by_flavor,
422        tags,
423        extras,
424        intersection,
425    ):
426        parts = ["try:"]
427
428        if platforms:
429            parts.extend(["-b", builds, "-p", ",".join(platforms)])
430
431        suites = tests if not intersection else {}
432        paths = set()
433        for flavor, flavor_tests in paths_by_flavor.items():
434            suite = self.flavor_suites[flavor]
435            if suite not in suites and (not intersection or suite in tests):
436                for job_name in self.flavor_jobs[flavor]:
437                    for test in flavor_tests:
438                        paths.add("{}:{}".format(flavor, test))
439                    suites[job_name] = tests.get(suite, [])
440
441        # intersection implies tests are expected
442        if intersection and not suites:
443            raise ValueError("No tests found matching filters")
444
445        if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]):
446            print(
447                'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. '
448                "Running without |--artifact| instead."
449            )
450            del extras["artifact"]
451
452        if extras.get("artifact"):
453            rejected = []
454            for suite in suites.keys():
455                if any([suite.startswith(c) for c in self.compiled_suites]):
456                    rejected.append(suite)
457            if rejected:
458                raise ValueError(
459                    "You can't run {} with "
460                    "--artifact option.".format(", ".join(rejected))
461                )
462
463        if extras.get("artifact") and "all" in suites.keys():
464            non_compiled_suites = set(self.common_suites) - set(self.compiled_suites)
465            message = (
466                "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})"
467                " can't run against an artifact build. Running (-u {non_compiled_suites}) "
468                "instead."
469            )
470            string_format = {
471                "tests": ",".join(self.compiled_suites),
472                "non_compiled_suites": ",".join(non_compiled_suites),
473            }
474            print(message.format(**string_format))
475            del suites["all"]
476            suites.update({suite_name: None for suite_name in non_compiled_suites})
477
478        if suites:
479            parts.append("-u")
480            parts.append(
481                ",".join(
482                    "{}{}".format(k, "[%s]" % ",".join(v) if v else "")
483                    for k, v in sorted(suites.items())
484                )
485            )
486
487        if talos:
488            parts.append("-t")
489            parts.append(
490                ",".join(
491                    "{}{}".format(k, "[%s]" % ",".join(v) if v else "")
492                    for k, v in sorted(talos.items())
493                )
494            )
495
496        if jobs:
497            parts.append("-j")
498            parts.append(",".join(jobs))
499
500        if tags:
501            parts.append(" ".join("--tag %s" % t for t in tags))
502
503        if paths:
504            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
505
506        args_by_dest = {
507            v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items()
508        }
509        for dest, value in extras.items():
510            assert dest in args_by_dest
511            arg = args_by_dest[dest]
512            action = SyntaxParser.pass_through_arguments[arg]["action"]
513            if action == "store":
514                parts.append(arg)
515                parts.append(value)
516            if action == "append":
517                for e in value:
518                    parts.append(arg)
519                    parts.append(e)
520            if action in ("store_true", "store_false"):
521                parts.append(arg)
522
523        return " ".join(parts)
524
525    def normalise_list(self, items, allow_subitems=False):
526        rv = defaultdict(list)
527        for item in items:
528            parsed = parse_arg(item)
529            for key, values in parsed.items():
530                rv[key].extend(values)
531
532        if not allow_subitems:
533            if not all(item == [] for item in rv.values()):
534                raise ValueError("Unexpected subitems in argument")
535            return rv.keys()
536        else:
537            return rv
538
539    def validate_args(self, **kwargs):
540        tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
541        if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
542            if "AUTOTRY_PLATFORM_HINT" in os.environ:
543                kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]]
544            elif tests_selected:
545                print("Must specify platform when selecting tests.")
546                sys.exit(1)
547            else:
548                print(
549                    "Either platforms or jobs must be specified as an argument to autotry."
550                )
551                sys.exit(1)
552
553        try:
554            platforms = (
555                self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {}
556            )
557        except ValueError as e:
558            print("Error parsing -p argument:\n%s" % e)
559            sys.exit(1)
560
561        try:
562            tests = (
563                self.normalise_list(kwargs["tests"], allow_subitems=True)
564                if kwargs["tests"]
565                else {}
566            )
567        except ValueError as e:
568            print("Error parsing -u argument ({}):\n{}".format(kwargs["tests"], e))
569            sys.exit(1)
570
571        try:
572            talos = (
573                self.normalise_list(kwargs["talos"], allow_subitems=True)
574                if kwargs["talos"]
575                else []
576            )
577        except ValueError as e:
578            print("Error parsing -t argument:\n%s" % e)
579            sys.exit(1)
580
581        try:
582            jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {}
583        except ValueError as e:
584            print("Error parsing -j argument:\n%s" % e)
585            sys.exit(1)
586
587        paths = []
588        for p in kwargs["paths"]:
589            p = mozpath.normpath(os.path.abspath(p))
590            if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
591                print(
592                    'Specified path "%s" is not a directory under the srcdir,'
593                    " unable to specify tests outside of the srcdir" % p
594                )
595                sys.exit(1)
596            if len(p) <= len(self.topsrcdir):
597                print(
598                    'Specified path "%s" is at the top of the srcdir and would'
599                    " select all tests." % p
600                )
601                sys.exit(1)
602            paths.append(os.path.relpath(p, self.topsrcdir))
603
604        try:
605            tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
606        except ValueError as e:
607            print("Error parsing --tags argument:\n%s" % e)
608            sys.exit(1)
609
610        extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()}
611        extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v}
612
613        return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
614
615    def run(self, **kwargs):
616        if not any(kwargs[item] for item in ("paths", "tests", "tags")):
617            kwargs["paths"] = set()
618            kwargs["tags"] = set()
619
620        builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(
621            **kwargs
622        )
623
624        if paths or tags:
625            paths = [
626                os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
627                for item in paths
628            ]
629            paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
630
631            if not paths_by_flavor and not tests:
632                print(
633                    "No tests were found when attempting to resolve paths:\n\n\t%s"
634                    % paths
635                )
636                sys.exit(1)
637
638            if not kwargs["intersection"]:
639                paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
640        else:
641            paths_by_flavor = {}
642
643        # No point in dealing with artifacts if we aren't running any builds
644        local_artifact_build = False
645        if platforms:
646            local_artifact_build = kwargs.get("local_artifact_build", False)
647
648            # Add --artifact if --enable-artifact-builds is set ...
649            if local_artifact_build:
650                extra["artifact"] = True
651            # ... unless --no-artifact is explicitly given.
652            if kwargs["no_artifact"]:
653                if "artifact" in extra:
654                    del extra["artifact"]
655
656        try:
657            msg = self.calc_try_syntax(
658                platforms,
659                tests,
660                talos,
661                jobs,
662                builds,
663                paths_by_flavor,
664                tags,
665                extra,
666                kwargs["intersection"],
667            )
668        except ValueError as e:
669            print(e)
670            sys.exit(1)
671
672        if local_artifact_build and not kwargs["no_artifact"]:
673            print(
674                "mozconfig has --enable-artifact-builds; including "
675                "--artifact flag in try syntax (use --no-artifact "
676                "to override)"
677            )
678
679        if kwargs["verbose"] and paths_by_flavor:
680            print("The following tests will be selected: ")
681            for flavor, paths in paths_by_flavor.items():
682                print("{}: {}".format(flavor, ",".join(paths)))
683
684        if kwargs["verbose"]:
685            print("The following try syntax was calculated:\n%s" % msg)
686
687        push_to_try(
688            "syntax",
689            kwargs["message"].format(msg=msg),
690            push=kwargs["push"],
691            closed_tree=kwargs["closed_tree"],
692        )
693
694
695def run(**kwargs):
696    at = AutoTry()
697    return at.run(**kwargs)
698