1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7import os
8import re
9import sys
10from collections import defaultdict
11
12import mozpack.path as mozpath
13import six
14from moztest.resolve import TestResolver
15
16from ..cli import BaseTryParser
17from ..push import build, push_to_try
18
19here = os.path.abspath(os.path.dirname(__file__))
20
21
22class SyntaxParser(BaseTryParser):
23    name = 'syntax'
24    arguments = [
25        [['paths'],
26         {'nargs': '*',
27          'default': [],
28          'help': 'Paths to search for tests to run on try.',
29          }],
30        [['-b', '--build'],
31         {'dest': 'builds',
32          'default': 'do',
33          'help': 'Build types to run (d for debug, o for optimized).',
34          }],
35        [['-p', '--platform'],
36         {'dest': 'platforms',
37          'action': 'append',
38          'help': 'Platforms to run (required if not found in the environment as '
39                  'AUTOTRY_PLATFORM_HINT).',
40          }],
41        [['-u', '--unittests'],
42         {'dest': 'tests',
43          'action': 'append',
44          'help': 'Test suites to run in their entirety.',
45          }],
46        [['-t', '--talos'],
47         {'action': 'append',
48          'help': 'Talos suites to run.',
49          }],
50        [['-j', '--jobs'],
51         {'action': 'append',
52          'help': 'Job tasks to run.',
53          }],
54        [['--tag'],
55         {'dest': 'tags',
56          'action': 'append',
57          'help': 'Restrict tests to the given tag (may be specified multiple times).',
58          }],
59        [['--and'],
60         {'action': 'store_true',
61          'dest': 'intersection',
62          'help': 'When -u and paths are supplied run only the intersection of the '
63                  'tests specified by the two arguments.',
64          }],
65        [['--no-artifact'],
66         {'action': 'store_true',
67          'help': 'Disable artifact builds even if --enable-artifact-builds is set '
68                  'in the mozconfig.',
69          }],
70        [['-v', '--verbose'],
71         {'dest': 'verbose',
72          'action': 'store_true',
73          'default': False,
74          'help': 'Print detailed information about the resulting test selection '
75                  'and commands performed.',
76          }],
77        [['--detect-paths'],
78         {'dest': 'detect_paths',
79          'action': 'store_true',
80          'default': False,
81          'help': 'Provide test paths based on files changed in the working copy.',
82          }],
83    ]
84
85    # Arguments we will accept on the command line and pass through to try
86    # syntax with no further intervention. The set is taken from
87    # http://trychooser.pub.build.mozilla.org with a few additions.
88    #
89    # Note that the meaning of store_false and store_true arguments is
90    # not preserved here, as we're only using these to echo the literal
91    # arguments to another consumer. Specifying either store_false or
92    # store_true here will have an equivalent effect.
93    pass_through_arguments = {
94        '--rebuild': {
95            'action': 'store',
96            'dest': 'rebuild',
97            'help': 'Re-trigger all test jobs (up to 20 times)',
98        },
99        '--rebuild-talos': {
100            'action': 'store',
101            'dest': 'rebuild_talos',
102            'help': 'Re-trigger all talos jobs',
103        },
104        '--interactive': {
105            'action': 'store_true',
106            'dest': 'interactive',
107            'help': 'Allow ssh-like access to running test containers',
108        },
109        '--no-retry': {
110            'action': 'store_true',
111            'dest': 'no_retry',
112            'help': 'Do not retrigger failed tests',
113        },
114        '--setenv': {
115            'action': 'append',
116            'dest': 'setenv',
117            'help': 'Set the corresponding variable in the test environment for '
118                    'applicable harnesses.',
119        },
120        '-f': {
121            'action': 'store_true',
122            'dest': 'failure_emails',
123            'help': 'Request failure emails only',
124        },
125        '--failure-emails': {
126            'action': 'store_true',
127            'dest': 'failure_emails',
128            'help': 'Request failure emails only',
129        },
130        '-e': {
131            'action': 'store_true',
132            'dest': 'all_emails',
133            'help': 'Request all emails',
134        },
135        '--all-emails': {
136            'action': 'store_true',
137            'dest': 'all_emails',
138            'help': 'Request all emails',
139        },
140        '--artifact': {
141            'action': 'store_true',
142            'dest': 'artifact',
143            'help': 'Force artifact builds where possible.',
144        },
145        '--upload-xdbs': {
146            'action': 'store_true',
147            'dest': 'upload_xdbs',
148            'help': 'Upload XDB compilation db files generated by hazard build',
149        },
150    }
151    task_configs = []
152
153    def __init__(self, *args, **kwargs):
154        BaseTryParser.__init__(self, *args, **kwargs)
155
156        group = self.add_argument_group("pass-through arguments")
157        for arg, opts in self.pass_through_arguments.items():
158            group.add_argument(arg, **opts)
159
160
161class TryArgumentTokenizer(object):
162    symbols = [("separator", ","),
163               ("list_start", "\["),
164               ("list_end", "\]"),
165               ("item", "([^,\[\]\s][^,\[\]]+)"),
166               ("space", "\s+")]
167    token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
168
169    def tokenize(self, data):
170        for match in self.token_re.finditer(data):
171            symbol = match.lastgroup
172            data = match.group(symbol)
173            if symbol == "space":
174                pass
175            else:
176                yield symbol, data
177
178
179class TryArgumentParser(object):
180    """Simple three-state parser for handling expressions
181    of the from "foo[sub item, another], bar,baz". This takes
182    input from the TryArgumentTokenizer and runs through a small
183    state machine, returning a dictionary of {top-level-item:[sub_items]}
184    i.e. the above would result in
185    {"foo":["sub item", "another"], "bar": [], "baz": []}
186    In the case of invalid input a ValueError is raised."""
187
188    EOF = object()
189
190    def __init__(self):
191        self.reset()
192
193    def reset(self):
194        self.tokens = None
195        self.current_item = None
196        self.data = {}
197        self.token = None
198        self.state = None
199
200    def parse(self, tokens):
201        self.reset()
202        self.tokens = tokens
203        self.consume()
204        self.state = self.item_state
205        while self.token[0] != self.EOF:
206            self.state()
207        return self.data
208
209    def consume(self):
210        try:
211            self.token = next(self.tokens)
212        except StopIteration:
213            self.token = (self.EOF, None)
214
215    def expect(self, *types):
216        if self.token[0] not in types:
217            raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
218
219    def item_state(self):
220        self.expect("item")
221        value = self.token[1].strip()
222        if value not in self.data:
223            self.data[value] = []
224        self.current_item = value
225        self.consume()
226        if self.token[0] == "separator":
227            self.consume()
228        elif self.token[0] == "list_start":
229            self.consume()
230            self.state = self.subitem_state
231        elif self.token[0] == self.EOF:
232            pass
233        else:
234            raise ValueError
235
236    def subitem_state(self):
237        self.expect("item")
238        value = self.token[1].strip()
239        self.data[self.current_item].append(value)
240        self.consume()
241        if self.token[0] == "separator":
242            self.consume()
243        elif self.token[0] == "list_end":
244            self.consume()
245            self.state = self.after_list_end_state
246        else:
247            raise ValueError
248
249    def after_list_end_state(self):
250        self.expect("separator")
251        self.consume()
252        self.state = self.item_state
253
254
255def parse_arg(arg):
256    tokenizer = TryArgumentTokenizer()
257    parser = TryArgumentParser()
258    return parser.parse(tokenizer.tokenize(arg))
259
260
261class AutoTry(object):
262
263    # Maps from flavors to the job names needed to run that flavour
264    flavor_jobs = {
265        'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
266        'xpcshell': ['xpcshell'],
267        'chrome': ['mochitest-o'],
268        'browser-chrome': ['mochitest-browser-chrome-1',
269                           'mochitest-e10s-browser-chrome-1',
270                           'mochitest-browser-chrome-e10s-1'],
271        'devtools-chrome': ['mochitest-devtools-chrome-1',
272                            'mochitest-e10s-devtools-chrome-1',
273                            'mochitest-devtools-chrome-e10s-1'],
274        'crashtest': ['crashtest', 'crashtest-e10s'],
275        'reftest': ['reftest', 'reftest-e10s'],
276        'remote': ['mochitest-remote'],
277        'web-platform-tests': ['web-platform-tests-1'],
278    }
279
280    flavor_suites = {
281        "mochitest": "mochitests",
282        "xpcshell": "xpcshell",
283        "chrome": "mochitest-o",
284        "browser-chrome": "mochitest-bc",
285        "devtools-chrome": "mochitest-dt",
286        "crashtest": "crashtest",
287        "reftest": "reftest",
288        "web-platform-tests": "web-platform-tests",
289    }
290
291    compiled_suites = [
292        "cppunit",
293        "gtest",
294        "jittest",
295    ]
296
297    common_suites = [
298        "cppunit",
299        "crashtest",
300        "firefox-ui-functional",
301        "geckoview",
302        "geckoview-junit",
303        "gtest",
304        "jittest",
305        "jsreftest",
306        "marionette",
307        "marionette-e10s",
308        "mochitests",
309        "reftest",
310        "robocop",
311        "web-platform-tests",
312        "xpcshell",
313    ]
314
315    def __init__(self):
316        self.topsrcdir = build.topsrcdir
317        self._resolver = None
318
319    @property
320    def resolver(self):
321        if self._resolver is None:
322            self._resolver = TestResolver.from_environment(cwd=here)
323        return self._resolver
324
325    @classmethod
326    def split_try_string(cls, data):
327        return re.findall(r'(?:\[.*?\]|\S)+', data)
328
329    def paths_by_flavor(self, paths=None, tags=None):
330        paths_by_flavor = defaultdict(set)
331
332        if not (paths or tags):
333            return dict(paths_by_flavor)
334
335        tests = list(self.resolver.resolve_tests(paths=paths,
336                                                 tags=tags))
337
338        for t in tests:
339            if t['flavor'] in self.flavor_suites:
340                flavor = t['flavor']
341                if 'subsuite' in t and t['subsuite'] == 'devtools':
342                    flavor = 'devtools-chrome'
343
344                if flavor in ['crashtest', 'reftest']:
345                    manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
346                    paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
347                elif 'dir_relpath' in t:
348                    paths_by_flavor[flavor].add(t['dir_relpath'])
349                else:
350                    file_relpath = os.path.relpath(t['path'], self.topsrcdir)
351                    dir_relpath = os.path.dirname(file_relpath)
352                    paths_by_flavor[flavor].add(dir_relpath)
353
354        for flavor, path_set in paths_by_flavor.items():
355            paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
356
357        return dict(paths_by_flavor)
358
359    def deduplicate_prefixes(self, path_set, input_paths):
360        # Removes paths redundant to test selection in the given path set.
361        # If a path was passed on the commandline that is the prefix of a
362        # path in our set, we only need to include the specified prefix to
363        # run the intended tests (every test in "layout/base" will run if
364        # "layout" is passed to the reftest harness).
365        removals = set()
366        additions = set()
367
368        for path in path_set:
369            full_path = path
370            while path:
371                path, _ = os.path.split(path)
372                if path in input_paths:
373                    removals.add(full_path)
374                    additions.add(path)
375
376        return additions | (path_set - removals)
377
378    def remove_duplicates(self, paths_by_flavor, tests):
379        rv = {}
380        for item in paths_by_flavor:
381            if self.flavor_suites[item] not in tests:
382                rv[item] = paths_by_flavor[item].copy()
383        return rv
384
385    def calc_try_syntax(self, platforms, tests, talos, jobs, builds, paths_by_flavor, tags,
386                        extras, intersection):
387        parts = ["try:"]
388
389        if platforms:
390            parts.extend(["-b", builds, "-p", ",".join(platforms)])
391
392        suites = tests if not intersection else {}
393        paths = set()
394        for flavor, flavor_tests in six.iteritems(paths_by_flavor):
395            suite = self.flavor_suites[flavor]
396            if suite not in suites and (not intersection or suite in tests):
397                for job_name in self.flavor_jobs[flavor]:
398                    for test in flavor_tests:
399                        paths.add("%s:%s" % (flavor, test))
400                    suites[job_name] = tests.get(suite, [])
401
402        # intersection implies tests are expected
403        if intersection and not suites:
404            raise ValueError("No tests found matching filters")
405
406        if extras.get('artifact') and any([p.endswith("-nightly") for p in platforms]):
407            print('You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. '
408                  'Running without |--artifact| instead.')
409            del extras['artifact']
410
411        if extras.get('artifact'):
412            rejected = []
413            for suite in suites.keys():
414                if any([suite.startswith(c) for c in self.compiled_suites]):
415                    rejected.append(suite)
416            if rejected:
417                raise ValueError("You can't run {} with "
418                                 "--artifact option.".format(', '.join(rejected)))
419
420        if extras.get('artifact') and 'all' in suites.keys():
421            non_compiled_suites = set(self.common_suites) - set(self.compiled_suites)
422            message = ('You asked for |-u all| with |--artifact| but compiled-code tests ({tests})'
423                       ' can\'t run against an artifact build. Running (-u {non_compiled_suites}) '
424                       'instead.')
425            string_format = {
426                'tests': ','.join(self.compiled_suites),
427                'non_compiled_suites': ','.join(non_compiled_suites),
428            }
429            print(message.format(**string_format))
430            del suites['all']
431            suites.update({suite_name: None for suite_name in non_compiled_suites})
432
433        if suites:
434            parts.append("-u")
435            parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
436                                  for k, v in sorted(suites.items())))
437
438        if talos:
439            parts.append("-t")
440            parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
441                                  for k, v in sorted(talos.items())))
442
443        if jobs:
444            parts.append("-j")
445            parts.append(",".join(jobs))
446
447        if tags:
448            parts.append(' '.join('--tag %s' % t for t in tags))
449
450        if paths:
451            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
452
453        args_by_dest = {v['dest']: k for k, v in SyntaxParser.pass_through_arguments.items()}
454        for dest, value in six.iteritems(extras):
455            assert dest in args_by_dest
456            arg = args_by_dest[dest]
457            action = SyntaxParser.pass_through_arguments[arg]['action']
458            if action == 'store':
459                parts.append(arg)
460                parts.append(value)
461            if action == 'append':
462                for e in value:
463                    parts.append(arg)
464                    parts.append(e)
465            if action in ('store_true', 'store_false'):
466                parts.append(arg)
467
468        return " ".join(parts)
469
470    def normalise_list(self, items, allow_subitems=False):
471        rv = defaultdict(list)
472        for item in items:
473            parsed = parse_arg(item)
474            for key, values in six.iteritems(parsed):
475                rv[key].extend(values)
476
477        if not allow_subitems:
478            if not all(item == [] for item in six.itervalues(rv)):
479                raise ValueError("Unexpected subitems in argument")
480            return rv.keys()
481        else:
482            return rv
483
484    def validate_args(self, **kwargs):
485        tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
486        if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
487            if 'AUTOTRY_PLATFORM_HINT' in os.environ:
488                kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
489            elif tests_selected:
490                print("Must specify platform when selecting tests.")
491                sys.exit(1)
492            else:
493                print("Either platforms or jobs must be specified as an argument to autotry.")
494                sys.exit(1)
495
496        try:
497            platforms = (self.normalise_list(kwargs["platforms"])
498                         if kwargs["platforms"] else {})
499        except ValueError as e:
500            print("Error parsing -p argument:\n%s" % e.message)
501            sys.exit(1)
502
503        try:
504            tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
505                     if kwargs["tests"] else {})
506        except ValueError as e:
507            print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
508            sys.exit(1)
509
510        try:
511            talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
512                     if kwargs["talos"] else [])
513        except ValueError as e:
514            print("Error parsing -t argument:\n%s" % e.message)
515            sys.exit(1)
516
517        try:
518            jobs = (self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {})
519        except ValueError as e:
520            print("Error parsing -j argument:\n%s" % e.message)
521            sys.exit(1)
522
523        paths = []
524        for p in kwargs["paths"]:
525            p = mozpath.normpath(os.path.abspath(p))
526            if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
527                print('Specified path "%s" is not a directory under the srcdir,'
528                      ' unable to specify tests outside of the srcdir' % p)
529                sys.exit(1)
530            if len(p) <= len(self.topsrcdir):
531                print('Specified path "%s" is at the top of the srcdir and would'
532                      ' select all tests.' % p)
533                sys.exit(1)
534            paths.append(os.path.relpath(p, self.topsrcdir))
535
536        try:
537            tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
538        except ValueError as e:
539            print("Error parsing --tags argument:\n%s" % e.message)
540            sys.exit(1)
541
542        extra_values = {k['dest'] for k in SyntaxParser.pass_through_arguments.values()}
543        extra_args = {k: v for k, v in kwargs.items()
544                      if k in extra_values and v}
545
546        return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
547
548    def run(self, **kwargs):
549        if not any(kwargs[item] for item in ("paths", "tests", "tags")):
550            if kwargs['detect_paths']:
551                res = self.resolver.get_outgoing_metadata()
552                kwargs['paths'] = res['paths']
553                kwargs['tags'] = res['tags']
554            else:
555                kwargs['paths'] = set()
556                kwargs['tags'] = set()
557
558        builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(**kwargs)
559
560        if paths or tags:
561            paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
562                     for item in paths]
563            paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
564
565            if not paths_by_flavor and not tests:
566                print("No tests were found when attempting to resolve paths:\n\n\t%s" %
567                      paths)
568                sys.exit(1)
569
570            if not kwargs["intersection"]:
571                paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
572        else:
573            paths_by_flavor = {}
574
575        # No point in dealing with artifacts if we aren't running any builds
576        local_artifact_build = False
577        if platforms:
578            local_artifact_build = kwargs.get('local_artifact_build', False)
579
580            # Add --artifact if --enable-artifact-builds is set ...
581            if local_artifact_build:
582                extra["artifact"] = True
583            # ... unless --no-artifact is explicitly given.
584            if kwargs["no_artifact"]:
585                if "artifact" in extra:
586                    del extra["artifact"]
587
588        try:
589            msg = self.calc_try_syntax(platforms, tests, talos, jobs, builds,
590                                       paths_by_flavor, tags, extra, kwargs["intersection"])
591        except ValueError as e:
592            print(e.message)
593            sys.exit(1)
594
595        if local_artifact_build and not kwargs["no_artifact"]:
596            print('mozconfig has --enable-artifact-builds; including '
597                  '--artifact flag in try syntax (use --no-artifact '
598                  'to override)')
599
600        if kwargs["verbose"] and paths_by_flavor:
601            print('The following tests will be selected: ')
602            for flavor, paths in six.iteritems(paths_by_flavor):
603                print("%s: %s" % (flavor, ",".join(paths)))
604
605        if kwargs["verbose"]:
606            print('The following try syntax was calculated:\n%s' % msg)
607
608        push_to_try('syntax', kwargs["message"].format(msg=msg), push=kwargs['push'],
609                    closed_tree=kwargs["closed_tree"])
610
611
612def run(**kwargs):
613    at = AutoTry()
614    return at.run(**kwargs)
615