1#!/usr/bin/env python
2
3"""
4Command line utility for executing MongoDB tests of all kinds.
5"""
6
7from __future__ import absolute_import
8
9import os.path
10import random
11import sys
12import time
13
14# Get relative imports to work when the package is not installed on the PYTHONPATH.
15if __name__ == "__main__" and __package__ is None:
16    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
17
18from buildscripts import resmokelib
19
20
21def _execute_suite(suite):
22    """
23    Executes the test suite, failing fast if requested.
24
25    Returns true if the execution of the suite was interrupted, and false otherwise.
26    """
27
28    logger = resmokelib.logging.loggers.EXECUTOR_LOGGER
29
30    if resmokelib.config.SHUFFLE:
31        logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.",
32                    suite.test_kind, suite.get_display_name(), resmokelib.config.RANDOM_SEED)
33        random.seed(resmokelib.config.RANDOM_SEED)
34        random.shuffle(suite.tests)
35
36    if resmokelib.config.DRY_RUN == "tests":
37        sb = []
38        sb.append("Tests that would be run for %ss in suite %s:"
39                  % (suite.test_kind, suite.get_display_name()))
40        if len(suite.tests) > 0:
41            for test in suite.tests:
42                sb.append(test)
43        else:
44            sb.append("(no tests)")
45        logger.info("\n".join(sb))
46
47        # Set a successful return code on the test suite because we want to output the tests
48        # that would get run by any other suites the user specified.
49        suite.return_code = 0
50        return False
51
52    if len(suite.tests) == 0:
53        logger.info("Skipping %ss, no tests to run", suite.test_kind)
54
55        # Set a successful return code on the test suite because we want to output the tests
56        # that would get run by any other suites the user specified.
57        suite.return_code = 0
58        return False
59
60    archive = None
61    if resmokelib.config.ARCHIVE_FILE:
62        archive = resmokelib.utils.archival.Archival(
63            archival_json_file=resmokelib.config.ARCHIVE_FILE,
64            limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB,
65            limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS,
66            logger=logger)
67
68    executor_config = suite.get_executor_config()
69
70    try:
71        executor = resmokelib.testing.executor.TestSuiteExecutor(
72            logger, suite, archive_instance=archive, **executor_config)
73        executor.run()
74        if suite.options.fail_fast and suite.return_code != 0:
75            return False
76    except (resmokelib.errors.UserInterrupt, resmokelib.errors.LoggerRuntimeConfigError) as err:
77        logger.error("Encountered an error when running %ss of suite %s: %s",
78                     suite.test_kind, suite.get_display_name(), err)
79        suite.return_code = err.EXIT_CODE
80        return True
81    except:
82        logger.exception("Encountered an error when running %ss of suite %s.",
83                         suite.test_kind, suite.get_display_name())
84        suite.return_code = 2
85        return False
86    finally:
87        if archive:
88            archive.exit()
89
90
91def _log_summary(logger, suites, time_taken):
92    if len(suites) > 1:
93        resmokelib.testing.suite.Suite.log_summaries(logger, suites, time_taken)
94
95
96def _summarize_suite(suite):
97    sb = []
98    suite.summarize(sb)
99    return "\n".join(sb)
100
101
102def _dump_suite_config(suite, logging_config):
103    """
104    Returns a string that represents the YAML configuration of a suite.
105
106    TODO: include the "options" key in the result
107    """
108
109    sb = []
110    sb.append("YAML configuration of suite %s" % (suite.get_display_name()))
111    sb.append(resmokelib.utils.dump_yaml({"test_kind": suite.get_test_kind_config()}))
112    sb.append("")
113    sb.append(resmokelib.utils.dump_yaml({"selector": suite.get_selector_config()}))
114    sb.append("")
115    sb.append(resmokelib.utils.dump_yaml({"executor": suite.get_executor_config()}))
116    sb.append("")
117    sb.append(resmokelib.utils.dump_yaml({"logging": logging_config}))
118    return "\n".join(sb)
119
120
121def find_suites_by_test(suites):
122    """
123    Looks up what other resmoke suites run the tests specified in the suites
124    parameter. Returns a dict keyed by test name, value is array of suite names.
125    """
126
127    memberships = {}
128    test_membership = resmokelib.parser.create_test_membership_map()
129    for suite in suites:
130        for test in suite.tests:
131            memberships[test] = test_membership[test]
132    return memberships
133
134
135def _list_suites_and_exit(logger, exit_code=0):
136    suite_names = resmokelib.parser.get_named_suites()
137    logger.info("Suites available to execute:\n%s", "\n".join(suite_names))
138    sys.exit(exit_code)
139
140
141class Main(object):
142    """
143    A class for executing potentially multiple resmoke.py test suites.
144    """
145
146    def __init__(self):
147        """
148        Initializes the Main instance by parsing the command line arguments.
149        """
150
151        self.__start_time = time.time()
152
153        values, args = resmokelib.parser.parse_command_line()
154        self.__values = values
155        self.__args = args
156
157    def _get_suites(self):
158        """
159        Returns a list of resmokelib.testing.suite.Suite instances to execute.
160        """
161
162        return resmokelib.parser.get_suites(self.__values, self.__args)
163
164    def run(self):
165        """
166        Executes the list of resmokelib.testing.suite.Suite instances returned by _get_suites().
167        """
168
169        logging_config = resmokelib.parser.get_logging_config(self.__values)
170        resmokelib.logging.loggers.configure_loggers(logging_config)
171        resmokelib.logging.flush.start_thread()
172
173        resmokelib.parser.update_config_vars(self.__values)
174
175        exec_logger = resmokelib.logging.loggers.EXECUTOR_LOGGER
176        resmoke_logger = exec_logger.new_resmoke_logger()
177
178        if self.__values.list_suites:
179            _list_suites_and_exit(resmoke_logger)
180
181        # Log the command line arguments specified to resmoke.py to make it easier to re-run the
182        # resmoke.py invocation used by an Evergreen task.
183        resmoke_logger.info("resmoke.py invocation: %s", " ".join(sys.argv))
184
185        interrupted = False
186        try:
187            suites = self._get_suites()
188        except resmokelib.errors.SuiteNotFound as err:
189            resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err))
190            _list_suites_and_exit(resmoke_logger, exit_code=1)
191
192        # Register a signal handler or Windows event object so we can write the report file if the
193        # task times out.
194        resmokelib.sighandler.register(resmoke_logger, suites, self.__start_time)
195
196        # Run the suite finder after the test suite parsing is complete.
197        if self.__values.find_suites:
198            suites_by_test = find_suites_by_test(suites)
199            for test in sorted(suites_by_test):
200                suite_names = suites_by_test[test]
201                resmoke_logger.info("%s will be run by the following suite(s): %s",
202                                    test, suite_names)
203            sys.exit(0)
204
205        exit_code = 0
206        try:
207            for suite in suites:
208                resmoke_logger.info(_dump_suite_config(suite, logging_config))
209
210                suite.record_suite_start()
211                interrupted = _execute_suite(suite)
212                suite.record_suite_end()
213
214                resmoke_logger.info("=" * 80)
215                resmoke_logger.info("Summary of %s suite: %s",
216                                    suite.get_display_name(), _summarize_suite(suite))
217
218                if interrupted or (suite.options.fail_fast and suite.return_code != 0):
219                    time_taken = time.time() - self.__start_time
220                    _log_summary(resmoke_logger, suites, time_taken)
221                    exit_code = suite.return_code
222                    sys.exit(exit_code)
223
224            time_taken = time.time() - self.__start_time
225            _log_summary(resmoke_logger, suites, time_taken)
226
227            # Exit with a nonzero code if any of the suites failed.
228            exit_code = max(suite.return_code for suite in suites)
229            sys.exit(exit_code)
230        finally:
231            # We want to exit as quickly as possible when interrupted by a user and therefore don't
232            # bother waiting for all log output to be flushed to logkeeper.
233            #
234            # If we already failed to write log output to logkeeper, then we don't bother waiting
235            # for any remaining log output to be flushed as it'll likely fail too. Exiting without
236            # joining the flush thread here also means that resmoke.py won't hang due a logger from
237            # a fixture or a background hook not being closed.
238            if not interrupted and not resmokelib.logging.buildlogger.is_log_output_incomplete():
239                resmokelib.logging.flush.stop_thread()
240
241            resmokelib.reportfile.write(suites)
242
243            if not interrupted and resmokelib.logging.buildlogger.is_log_output_incomplete():
244                if exit_code == 0:
245                    # We don't anticipate users to look at passing Evergreen tasks very often that
246                    # even if the log output is incomplete, we'd still rather not show anything in
247                    # the Evergreen UI or cause a JIRA ticket to be created.
248                    resmoke_logger.info(
249                        "We failed to flush all log output to logkeeper but all tests passed, so"
250                        " ignoring.")
251                else:
252                    resmoke_logger.info(
253                        "Exiting with code %d rather than requested code %d because we failed to"
254                        " flush all log output to logkeeper.",
255                        resmokelib.errors.LoggerRuntimeConfigError.EXIT_CODE, exit_code)
256                    sys.exit(resmokelib.errors.LoggerRuntimeConfigError.EXIT_CODE)
257
258if __name__ == "__main__":
259    Main().run()
260