1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5import argparse
6import itertools
7import os
8import re
9import subprocess
10import sys
11import which
12
13from collections import defaultdict
14
15import ConfigParser
16
17
18def arg_parser():
19    parser = argparse.ArgumentParser()
20    parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
21    parser.add_argument('-b', '--build', dest='builds', default='do',
22                        help='Build types to run (d for debug, o for optimized).')
23    parser.add_argument('-p', '--platform', dest='platforms', action='append',
24                        help='Platforms to run (required if not found in the environment as AUTOTRY_PLATFORM_HINT).')
25    parser.add_argument('-u', '--unittests', dest='tests', action='append',
26                        help='Test suites to run in their entirety.')
27    parser.add_argument('-t', '--talos', dest='talos', action='append',
28                        help='Talos suites to run.')
29    parser.add_argument('--tag', dest='tags', action='append',
30                        help='Restrict tests to the given tag (may be specified multiple times).')
31    parser.add_argument('--and', action='store_true', dest='intersection',
32                        help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments.')
33    parser.add_argument('--no-push', dest='push', action='store_false',
34                        help='Do not push to try as a result of running this command (if '
35                        'specified this command will only print calculated try '
36                        'syntax and selection info).')
37    parser.add_argument('--save', dest='save', action='store',
38                        help='Save the command line arguments for future use with --preset.')
39    parser.add_argument('--preset', dest='load', action='store',
40                        help='Load a saved set of arguments. Additional arguments will override saved ones.')
41    parser.add_argument('--list', action='store_true',
42                        help='List all saved try strings')
43    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False,
44                        help='Print detailed information about the resulting test selection '
45                        'and commands performed.')
46    for arg, opts in AutoTry.pass_through_arguments.items():
47        parser.add_argument(arg, **opts)
48    return parser
49
50class TryArgumentTokenizer(object):
51    symbols = [("seperator", ","),
52               ("list_start", "\["),
53               ("list_end", "\]"),
54               ("item", "([^,\[\]\s][^,\[\]]+)"),
55               ("space", "\s+")]
56    token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
57
58    def tokenize(self, data):
59        for match in self.token_re.finditer(data):
60            symbol = match.lastgroup
61            data = match.group(symbol)
62            if symbol == "space":
63                pass
64            else:
65                yield symbol, data
66
67class TryArgumentParser(object):
68    """Simple three-state parser for handling expressions
69    of the from "foo[sub item, another], bar,baz". This takes
70    input from the TryArgumentTokenizer and runs through a small
71    state machine, returning a dictionary of {top-level-item:[sub_items]}
72    i.e. the above would result in
73    {"foo":["sub item", "another"], "bar": [], "baz": []}
74    In the case of invalid input a ValueError is raised."""
75
76    EOF = object()
77
78    def __init__(self):
79        self.reset()
80
81    def reset(self):
82        self.tokens = None
83        self.current_item = None
84        self.data = {}
85        self.token = None
86        self.state = None
87
88    def parse(self, tokens):
89        self.reset()
90        self.tokens = tokens
91        self.consume()
92        self.state = self.item_state
93        while self.token[0] != self.EOF:
94            self.state()
95        return self.data
96
97    def consume(self):
98        try:
99            self.token = self.tokens.next()
100        except StopIteration:
101            self.token = (self.EOF, None)
102
103    def expect(self, *types):
104        if self.token[0] not in types:
105            raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
106
107    def item_state(self):
108        self.expect("item")
109        value = self.token[1].strip()
110        if value not in self.data:
111            self.data[value] = []
112        self.current_item = value
113        self.consume()
114        if self.token[0] == "seperator":
115            self.consume()
116        elif self.token[0] == "list_start":
117            self.consume()
118            self.state = self.subitem_state
119        elif self.token[0] == self.EOF:
120            pass
121        else:
122            raise ValueError
123
124    def subitem_state(self):
125        self.expect("item")
126        value = self.token[1].strip()
127        self.data[self.current_item].append(value)
128        self.consume()
129        if self.token[0] == "seperator":
130            self.consume()
131        elif self.token[0] == "list_end":
132            self.consume()
133            self.state = self.after_list_end_state
134        else:
135            raise ValueError
136
137    def after_list_end_state(self):
138        self.expect("seperator")
139        self.consume()
140        self.state = self.item_state
141
142def parse_arg(arg):
143    tokenizer = TryArgumentTokenizer()
144    parser = TryArgumentParser()
145    return parser.parse(tokenizer.tokenize(arg))
146
147class AutoTry(object):
148
149    # Maps from flavors to the job names needed to run that flavour
150    flavor_jobs = {
151        'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
152        'xpcshell': ['xpcshell'],
153        'chrome': ['mochitest-o'],
154        'browser-chrome': ['mochitest-browser-chrome-1',
155                           'mochitest-e10s-browser-chrome-1'],
156        'devtools-chrome': ['mochitest-devtools-chrome-1',
157                            'mochitest-e10s-devtools-chrome-1'],
158        'crashtest': ['crashtest', 'crashtest-e10s'],
159        'reftest': ['reftest', 'reftest-e10s'],
160        'web-platform-tests': ['web-platform-tests-1'],
161    }
162
163    flavor_suites = {
164        "mochitest": "mochitests",
165        "xpcshell": "xpcshell",
166        "chrome": "mochitest-o",
167        "browser-chrome": "mochitest-bc",
168        "devtools-chrome": "mochitest-dt",
169        "crashtest": "crashtest",
170        "reftest": "reftest",
171        "web-platform-tests": "web-platform-tests",
172    }
173
174    compiled_suites = [
175        "cppunit",
176        "gtest",
177        "jittest",
178    ]
179
180    common_suites = [
181        "cppunit",
182        "crashtest",
183        "firefox-ui-functional",
184        "gtest",
185        "jittest",
186        "jsreftest",
187        "marionette",
188        "marionette-e10s",
189        "media-tests",
190        "mochitests",
191        "reftest",
192        "web-platform-tests",
193        "xpcshell",
194    ]
195
196    # Arguments we will accept on the command line and pass through to try
197    # syntax with no further intervention. The set is taken from
198    # http://trychooser.pub.build.mozilla.org with a few additions.
199    #
200    # Note that the meaning of store_false and store_true arguments is
201    # not preserved here, as we're only using these to echo the literal
202    # arguments to another consumer. Specifying either store_false or
203    # store_true here will have an equivalent effect.
204    pass_through_arguments = {
205        '--rebuild': {
206            'action': 'store',
207            'dest': 'rebuild',
208            'help': 'Re-trigger all test jobs (up to 20 times)',
209        },
210        '--rebuild-talos': {
211            'action': 'store',
212            'dest': 'rebuild_talos',
213            'help': 'Re-trigger all talos jobs',
214        },
215        '--interactive': {
216            'action': 'store_true',
217            'dest': 'interactive',
218            'help': 'Allow ssh-like access to running test containers',
219        },
220        '--no-retry': {
221            'action': 'store_true',
222            'dest': 'no_retry',
223            'help': 'Do not retrigger failed tests',
224        },
225        '--setenv': {
226            'action': 'append',
227            'dest': 'setenv',
228            'help': 'Set the corresponding variable in the test environment for'
229                    'applicable harnesses.',
230        },
231        '-f': {
232            'action': 'store_true',
233            'dest': 'failure_emails',
234            'help': 'Request failure emails only',
235        },
236        '--failure-emails': {
237            'action': 'store_true',
238            'dest': 'failure_emails',
239            'help': 'Request failure emails only',
240        },
241        '-e': {
242            'action': 'store_true',
243            'dest': 'all_emails',
244            'help': 'Request all emails',
245        },
246        '--all-emails': {
247            'action': 'store_true',
248            'dest': 'all_emails',
249            'help': 'Request all emails',
250        },
251        '--artifact': {
252            'action': 'store_true',
253            'dest': 'artifact',
254            'help': 'Force artifact builds where possible.',
255        }
256    }
257
258    def __init__(self, topsrcdir, resolver_func, mach_context):
259        self.topsrcdir = topsrcdir
260        self._resolver_func = resolver_func
261        self._resolver = None
262        self.mach_context = mach_context
263
264        if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
265            self._use_git = False
266        else:
267            self._use_git = True
268
269    @property
270    def resolver(self):
271        if self._resolver is None:
272            self._resolver = self._resolver_func()
273        return self._resolver
274
275    @property
276    def config_path(self):
277        return os.path.join(self.mach_context.state_dir, "autotry.ini")
278
279    def load_config(self, name):
280        config = ConfigParser.RawConfigParser()
281        success = config.read([self.config_path])
282        if not success:
283            return None
284
285        try:
286            data = config.get("try", name)
287        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
288            return None
289
290        kwargs = vars(arg_parser().parse_args(self.split_try_string(data)))
291
292        return kwargs
293
294    def list_presets(self):
295        config = ConfigParser.RawConfigParser()
296        success = config.read([self.config_path])
297
298        data = []
299        if success:
300            try:
301                data = config.items("try")
302            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
303                pass
304
305        if not data:
306            print("No presets found")
307
308        for name, try_string in data:
309            print("%s: %s" % (name, try_string))
310
311    def split_try_string(self, data):
312        return re.findall(r'(?:\[.*?\]|\S)+', data)
313
314    def save_config(self, name, data):
315        assert data.startswith("try: ")
316        data = data[len("try: "):]
317
318        parser = ConfigParser.RawConfigParser()
319        parser.read([self.config_path])
320
321        if not parser.has_section("try"):
322            parser.add_section("try")
323
324        parser.set("try", name, data)
325
326        with open(self.config_path, "w") as f:
327            parser.write(f)
328
329    def paths_by_flavor(self, paths=None, tags=None):
330        paths_by_flavor = defaultdict(set)
331
332        if not (paths or tags):
333            return dict(paths_by_flavor)
334
335        tests = list(self.resolver.resolve_tests(paths=paths,
336                                                 tags=tags))
337
338        for t in tests:
339            if t['flavor'] in self.flavor_suites:
340                flavor = t['flavor']
341                if 'subsuite' in t and t['subsuite'] == 'devtools':
342                    flavor = 'devtools-chrome'
343
344                if flavor in ['crashtest', 'reftest']:
345                    manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
346                    paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
347                elif 'dir_relpath' in t:
348                    paths_by_flavor[flavor].add(t['dir_relpath'])
349                else:
350                    file_relpath = os.path.relpath(t['path'], self.topsrcdir)
351                    dir_relpath = os.path.dirname(file_relpath)
352                    paths_by_flavor[flavor].add(dir_relpath)
353
354        for flavor, path_set in paths_by_flavor.items():
355            paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
356
357        return dict(paths_by_flavor)
358
359    def deduplicate_prefixes(self, path_set, input_paths):
360        # Removes paths redundant to test selection in the given path set.
361        # If a path was passed on the commandline that is the prefix of a
362        # path in our set, we only need to include the specified prefix to
363        # run the intended tests (every test in "layout/base" will run if
364        # "layout" is passed to the reftest harness).
365        removals = set()
366        additions = set()
367
368        for path in path_set:
369            full_path = path
370            while path:
371                path, _ = os.path.split(path)
372                if path in input_paths:
373                    removals.add(full_path)
374                    additions.add(path)
375
376        return additions | (path_set - removals)
377
378    def remove_duplicates(self, paths_by_flavor, tests):
379        rv = {}
380        for item in paths_by_flavor:
381            if self.flavor_suites[item] not in tests:
382                rv[item] = paths_by_flavor[item].copy()
383        return rv
384
385    def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags,
386                        extras, intersection):
387        parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
388
389        suites = tests if not intersection else {}
390        paths = set()
391        for flavor, flavor_tests in paths_by_flavor.iteritems():
392            suite = self.flavor_suites[flavor]
393            if suite not in suites and (not intersection or suite in tests):
394                for job_name in self.flavor_jobs[flavor]:
395                    for test in flavor_tests:
396                        paths.add("%s:%s" % (flavor, test))
397                    suites[job_name] = tests.get(suite, [])
398
399        if not suites:
400            raise ValueError("No tests found matching filters")
401
402        if extras.get('artifact'):
403            rejected = []
404            for suite in suites.keys():
405                if any([suite.startswith(c) for c in self.compiled_suites]):
406                    rejected.append(suite)
407            if rejected:
408                raise ValueError("You can't run {} with "
409                                 "--artifact option.".format(', '.join(rejected)))
410
411        parts.append("-u")
412        parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
413                              for k,v in sorted(suites.items())) if suites else "none")
414
415        parts.append("-t")
416        parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
417                              for k,v in sorted(talos.items())) if talos else "none")
418
419        if tags:
420            parts.append(' '.join('--tag %s' % t for t in tags))
421
422        if paths:
423            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
424
425        args_by_dest = {v['dest']: k for k, v in AutoTry.pass_through_arguments.items()}
426        for dest, value in extras.iteritems():
427            assert dest in args_by_dest
428            arg = args_by_dest[dest]
429            action = AutoTry.pass_through_arguments[arg]['action']
430            if action == 'store':
431                parts.append(arg)
432                parts.append(value)
433            if action == 'append':
434                for e in value:
435                    parts.append(arg)
436                    parts.append(e)
437            if action in ('store_true', 'store_false'):
438                parts.append(arg)
439
440        try_syntax = " ".join(parts)
441        if extras.get('artifact') and 'all' in suites.keys():
442            message = ('You asked for |-u all| with |--artifact| but compiled-code tests ({tests})'
443                       ' can\'t run against an artifact build. Try listing the suites you want'
444                       ' instead. For example, this syntax covers most suites:\n{try_syntax}')
445            string_format = {
446                'tests': ','.join(self.compiled_suites),
447                'try_syntax': try_syntax.replace(
448                    '-u all',
449                    '-u ' + ','.join(sorted(set(self.common_suites) - set(self.compiled_suites)))
450                )
451            }
452            raise ValueError(message.format(**string_format))
453
454        return try_syntax
455
456    def _run_git(self, *args):
457        args = ['git'] + list(args)
458        ret = subprocess.call(args)
459        if ret:
460            print('ERROR git command %s returned %s' %
461                  (args, ret))
462            sys.exit(1)
463
464    def _git_push_to_try(self, msg):
465        self._run_git('commit', '--allow-empty', '-m', msg)
466        try:
467            self._run_git('push', 'hg::ssh://hg.mozilla.org/try',
468                          '+HEAD:refs/heads/branches/default/tip')
469        finally:
470            self._run_git('reset', 'HEAD~')
471
472    def _git_find_changed_files(self):
473        # This finds the files changed on the current branch based on the
474        # diff of the current branch its merge-base base with other branches.
475        try:
476            args = ['git', 'rev-parse', 'HEAD']
477            current_branch = subprocess.check_output(args).strip()
478            args = ['git', 'for-each-ref', 'refs/heads', 'refs/remotes',
479                    '--format=%(objectname)']
480            all_branches = subprocess.check_output(args).splitlines()
481            other_branches = set(all_branches) - set([current_branch])
482            args = ['git', 'merge-base', 'HEAD'] + list(other_branches)
483            base_commit = subprocess.check_output(args).strip()
484            args = ['git', 'diff', '--name-only', '-z', 'HEAD', base_commit]
485            return subprocess.check_output(args).strip('\0').split('\0')
486        except subprocess.CalledProcessError as e:
487            print('Failed while determining files changed on this branch')
488            print('Failed whle running: %s' % args)
489            print(e.output)
490            sys.exit(1)
491
492    def _hg_find_changed_files(self):
493        hg_args = [
494            'hg', 'log', '-r',
495            '::. and not public()',
496            '--template',
497            '{join(files, "\n")}\n',
498        ]
499        try:
500            return subprocess.check_output(hg_args).splitlines()
501        except subprocess.CalledProcessError as e:
502            print('Failed while finding files changed since the last '
503                  'public ancestor')
504            print('Failed whle running: %s' % hg_args)
505            print(e.output)
506            sys.exit(1)
507
508    def find_changed_files(self):
509        """Finds files changed in a local source tree.
510
511        For hg, changes since the last public ancestor of '.' are
512        considered. For git, changes in the current branch are considered.
513        """
514        if self._use_git:
515            return self._git_find_changed_files()
516        return self._hg_find_changed_files()
517
518    def push_to_try(self, msg, verbose):
519        if not self._use_git:
520            try:
521                hg_args = ['hg', 'push-to-try', '-m', msg]
522                subprocess.check_call(hg_args, stderr=subprocess.STDOUT)
523            except subprocess.CalledProcessError as e:
524                print('ERROR hg command %s returned %s' % (hg_args, e.returncode))
525                print('\nmach failed to push to try. There may be a problem '
526                      'with your ssh key, or another issue with your mercurial '
527                      'installation.')
528                # Check for the presence of the "push-to-try" extension, and
529                # provide instructions if it can't be found.
530                try:
531                    subprocess.check_output(['hg', 'showconfig',
532                                             'extensions.push-to-try'])
533                except subprocess.CalledProcessError:
534                    print('\nThe "push-to-try" hg extension is required. It '
535                          'can be installed to Mercurial 3.3 or above by '
536                          'running ./mach mercurial-setup')
537                sys.exit(1)
538        else:
539            try:
540                which.which('git-cinnabar')
541                self._git_push_to_try(msg)
542            except which.WhichError:
543                print('ERROR git-cinnabar is required to push from git to try with'
544                      'the autotry command.\n\nMore information can by found at '
545                      'https://github.com/glandium/git-cinnabar')
546                sys.exit(1)
547
548    def find_uncommited_changes(self):
549        if self._use_git:
550            stat = subprocess.check_output(['git', 'status', '-z'])
551            return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'D')
552                       for entry in stat.split('\0'))
553        else:
554            stat = subprocess.check_output(['hg', 'status'])
555            return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'R')
556                       for entry in stat.splitlines())
557
558    def find_paths_and_tags(self, verbose):
559        paths, tags = set(), set()
560        changed_files = self.find_changed_files()
561        if changed_files:
562            if verbose:
563                print("Pushing tests based on modifications to the "
564                      "following files:\n\t%s" % "\n\t".join(changed_files))
565
566            from mozbuild.frontend.reader import (
567                BuildReader,
568                EmptyConfig,
569            )
570
571            config = EmptyConfig(self.topsrcdir)
572            reader = BuildReader(config)
573            files_info = reader.files_info(changed_files)
574
575            for path, info in files_info.items():
576                paths |= info.test_files
577                tags |= info.test_tags
578
579            if verbose:
580                if paths:
581                    print("Pushing tests based on the following patterns:\n\t%s" %
582                          "\n\t".join(paths))
583                if tags:
584                    print("Pushing tests based on the following tags:\n\t%s" %
585                          "\n\t".join(tags))
586        return paths, tags
587