1"""
2Holder for the (test kind, list of tests) pair with additional metadata about when and how they
3execute.
4"""
5
6from __future__ import absolute_import
7
8import itertools
9import threading
10import time
11
12from . import report as _report
13from . import summary as _summary
14from .. import config as _config
15from .. import selector as _selector
16
17
18def synchronized(method):
19    """Decorator to enfore instance lock ownership when calling the method."""
20
21    def synced(self, *args, **kwargs):
22        lock = getattr(self, "_lock")
23        with lock:
24            return method(self, *args, **kwargs)
25
26    return synced
27
28
29class Suite(object):
30    """
31    A suite of tests of a particular kind (e.g. C++ unit tests, dbtests, jstests).
32    """
33
34    def __init__(self, suite_name, suite_config, suite_options=_config.SuiteOptions.ALL_INHERITED):
35        """
36        Initializes the suite with the specified name and configuration.
37        """
38        self._lock = threading.RLock()
39
40        self._suite_name = suite_name
41        self._suite_config = suite_config
42        self._suite_options = suite_options
43
44        self.test_kind = self.get_test_kind_config()
45        self.tests = self._get_tests_for_kind(self.test_kind)
46
47        self.return_code = None  # Set by the executor.
48
49        self._suite_start_time = None
50        self._suite_end_time = None
51
52        self._test_start_times = []
53        self._test_end_times = []
54        self._reports = []
55
56        # We keep a reference to the TestReports from the currently running jobs so that we can
57        # report intermediate results.
58        self._partial_reports = None
59
60    def _get_tests_for_kind(self, test_kind):
61        """
62        Returns the tests to run based on the 'test_kind'-specific
63        filtering policy.
64        """
65        test_info = self.get_selector_config()
66
67        # The mongos_test doesn't have to filter anything, the test_info is just the arguments to
68        # the mongos program to be used as the test case.
69        if test_kind == "mongos_test":
70            mongos_options = test_info  # Just for easier reading.
71            if not isinstance(mongos_options, dict):
72                raise TypeError("Expected dictionary of arguments to mongos")
73            return [mongos_options]
74
75        tests = _selector.filter_tests(test_kind, test_info)
76        if _config.ORDER_TESTS_BY_NAME:
77            return sorted(tests, key=str.lower)
78
79        return tests
80
81    def get_name(self):
82        """
83        Returns the name of the test suite.
84        """
85        return self._suite_name
86
87    def get_display_name(self):
88        """
89        Returns the name of the test suite with a unique identifier for its SuiteOptions.
90        """
91
92        if self.options.description is None:
93            return self.get_name()
94
95        return "{} ({})".format(self.get_name(), self.options.description)
96
97    def get_selector_config(self):
98        """
99        Returns the "selector" section of the YAML configuration.
100        """
101
102        selector = self._suite_config["selector"].copy()
103
104        if self.options.include_tags is not None:
105            if "include_tags" in selector:
106                selector["include_tags"] = {"$allOf": [
107                    selector["include_tags"],
108                    self.options.include_tags,
109                ]}
110            elif "exclude_tags" in selector:
111                selector["exclude_tags"] = {"$anyOf": [
112                    selector["exclude_tags"],
113                    {"$not": self.options.include_tags},
114                ]}
115            else:
116                selector["include_tags"] = self.options.include_tags
117
118        return selector
119
120    def get_executor_config(self):
121        """
122        Returns the "executor" section of the YAML configuration.
123        """
124        return self._suite_config["executor"]
125
126    def get_test_kind_config(self):
127        """
128        Returns the "test_kind" section of the YAML configuration.
129        """
130        return self._suite_config["test_kind"]
131
132    @property
133    def options(self):
134        return self._suite_options.resolve()
135
136    def with_options(self, suite_options):
137        """
138        Returns a Suite instance with the specified resmokelib.config.SuiteOptions.
139        """
140
141        return Suite(self._suite_name, self._suite_config, suite_options)
142
143    @synchronized
144    def record_suite_start(self):
145        """
146        Records the start time of the suite.
147        """
148        self._suite_start_time = time.time()
149
150    @synchronized
151    def record_suite_end(self):
152        """
153        Records the end time of the suite.
154        """
155        self._suite_end_time = time.time()
156
157    @synchronized
158    def record_test_start(self, partial_reports):
159        """
160        Records the start time of an execution and stores the
161        TestReports for currently running jobs.
162        """
163        self._test_start_times.append(time.time())
164        self._partial_reports = partial_reports
165
166    @synchronized
167    def record_test_end(self, report):
168        """
169        Records the end time of an execution.
170        """
171        self._test_end_times.append(time.time())
172        self._reports.append(report)
173        self._partial_reports = None
174
175    @synchronized
176    def get_active_report(self):
177        """
178        Returns the partial report of the currently running execution, if there is one.
179        """
180        if not self._partial_reports:
181            return None
182        return _report.TestReport.combine(*self._partial_reports)
183
184    @synchronized
185    def get_reports(self):
186        """
187        Returns the list of reports. If there's an execution currently
188        in progress, then a report for the partial results is included
189        in the returned list.
190        """
191
192        if self._partial_reports is not None:
193            return self._reports + [self.get_active_report()]
194
195        return self._reports
196
197    @synchronized
198    def summarize(self, sb):
199        """
200        Appends a summary of the suite onto the string builder 'sb'.
201        """
202        if not self._reports and not self._partial_reports:
203            sb.append("No tests ran.")
204            summary = _summary.Summary(0, 0.0, 0, 0, 0, 0)
205        elif not self._reports and self._partial_reports:
206            summary = self.summarize_latest(sb)
207        elif len(self._reports) == 1 and not self._partial_reports:
208            summary = self._summarize_execution(0, sb)
209        else:
210            summary = self._summarize_repeated(sb)
211
212        summarized_group = "    %ss: %s" % (self.test_kind, "\n        ".join(sb))
213
214        if summary.num_run == 0:
215            sb.append("Suite did not run any tests.")
216            return
217
218        # Override the 'time_taken' attribute of the summary if we have more accurate timing
219        # information available.
220        if self._suite_start_time is not None and self._suite_end_time is not None:
221            time_taken = self._suite_end_time - self._suite_start_time
222            summary = summary._replace(time_taken=time_taken)
223
224        sb.append("%d test(s) ran in %0.2f seconds"
225                  " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary)
226
227        sb.append(summarized_group)
228
229    @synchronized
230    def summarize_latest(self, sb):
231        """
232        Returns a summary of the latest execution of the suite and appends a
233        summary of that execution onto the string builder 'sb'.
234
235        If there's an execution currently in progress, then the partial
236        summary of that execution is appended to 'sb'.
237        """
238
239        if self._partial_reports is None:
240            return self._summarize_execution(-1, sb)
241
242        active_report = _report.TestReport.combine(*self._partial_reports)
243        # Use the current time as the time that this suite finished running.
244        end_time = time.time()
245        return self._summarize_report(active_report, self._test_start_times[-1], end_time, sb)
246
247    def _summarize_repeated(self, sb):
248        """
249        Returns the summary information of all executions and appends
250        each execution's summary onto the string builder 'sb'. Also
251        appends information of how many repetitions there were.
252        """
253
254        reports = self.get_reports()  # Also includes the combined partial reports.
255        num_iterations = len(reports)
256        start_times = self._test_start_times[:]
257        end_times = self._test_end_times[:]
258        if self._partial_reports:
259            end_times.append(time.time())  # Add an end time in this copy for the partial reports.
260
261        total_time_taken = end_times[-1] - start_times[0]
262        sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken))
263
264        combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0)
265        for iteration in xrange(num_iterations):
266            # Summarize each execution as a bulleted list of results.
267            bulleter_sb = []
268            summary = self._summarize_report(
269                reports[iteration],
270                start_times[iteration],
271                end_times[iteration],
272                bulleter_sb)
273            combined_summary = _summary.combine(combined_summary, summary)
274
275            for (i, line) in enumerate(bulleter_sb):
276                # Only bullet first line, indent others.
277                prefix = "* " if i == 0 else "  "
278                sb.append(prefix + line)
279
280        return combined_summary
281
282    def _summarize_execution(self, iteration, sb):
283        """
284        Returns the summary information of the execution given by
285        'iteration' and appends a summary of that execution onto the
286        string builder 'sb'.
287        """
288
289        return self._summarize_report(self._reports[iteration],
290                                      self._test_start_times[iteration],
291                                      self._test_end_times[iteration],
292                                      sb)
293
294    def _summarize_report(self, report, start_time, end_time, sb):
295        """
296        Returns the summary information of the execution given by
297        'report' that started at 'start_time' and finished at
298        'end_time', and appends a summary of that execution onto the
299        string builder 'sb'.
300        """
301
302        time_taken = end_time - start_time
303
304        # Tests that were interrupted are treated as failures because (1) the test has already been
305        # started and therefore isn't skipped and (2) the test has yet to finish and therefore
306        # cannot be said to have succeeded.
307        num_failed = report.num_failed + report.num_interrupted
308        num_run = report.num_succeeded + report.num_errored + num_failed
309        num_skipped = len(self.tests) + report.num_dynamic - num_run
310
311        if report.num_succeeded == num_run and num_skipped == 0:
312            sb.append("All %d test(s) passed in %0.2f seconds." % (num_run, time_taken))
313            return _summary.Summary(num_run, time_taken, num_run, 0, 0, 0)
314
315        summary = _summary.Summary(num_run, time_taken, report.num_succeeded, num_skipped,
316                                   num_failed, report.num_errored)
317
318        sb.append("%d test(s) ran in %0.2f seconds"
319                  " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary)
320
321        if num_failed > 0:
322            sb.append("The following tests failed (with exit code):")
323            for test_info in itertools.chain(report.get_failed(), report.get_interrupted()):
324                sb.append("    %s (%d)" % (test_info.test_id, test_info.return_code))
325
326        if report.num_errored > 0:
327            sb.append("The following tests had errors:")
328            for test_info in report.get_errored():
329                sb.append("    %s" % (test_info.test_id))
330
331        return summary
332
333    @staticmethod
334    def log_summaries(logger, suites, time_taken):
335        sb = []
336        sb.append("Summary of all suites: %d suites ran in %0.2f seconds"
337                  % (len(suites), time_taken))
338        for suite in suites:
339            suite_sb = []
340            suite.summarize(suite_sb)
341            sb.append("    %s: %s" % (suite.get_display_name(), "\n    ".join(suite_sb)))
342
343        logger.info("=" * 80)
344        logger.info("\n".join(sb))
345