1#!/usr/bin/env python 2 3""" 4Command line utility for executing MongoDB tests of all kinds. 5""" 6 7from __future__ import absolute_import 8 9import os.path 10import random 11import sys 12import time 13 14# Get relative imports to work when the package is not installed on the PYTHONPATH. 15if __name__ == "__main__" and __package__ is None: 16 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 17 18from buildscripts import resmokelib 19 20 21def _execute_suite(suite): 22 """ 23 Executes the test suite, failing fast if requested. 24 25 Returns true if the execution of the suite was interrupted, and false otherwise. 26 """ 27 28 logger = resmokelib.logging.loggers.EXECUTOR_LOGGER 29 30 if resmokelib.config.SHUFFLE: 31 logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.", 32 suite.test_kind, suite.get_display_name(), resmokelib.config.RANDOM_SEED) 33 random.seed(resmokelib.config.RANDOM_SEED) 34 random.shuffle(suite.tests) 35 36 if resmokelib.config.DRY_RUN == "tests": 37 sb = [] 38 sb.append("Tests that would be run for %ss in suite %s:" 39 % (suite.test_kind, suite.get_display_name())) 40 if len(suite.tests) > 0: 41 for test in suite.tests: 42 sb.append(test) 43 else: 44 sb.append("(no tests)") 45 logger.info("\n".join(sb)) 46 47 # Set a successful return code on the test suite because we want to output the tests 48 # that would get run by any other suites the user specified. 49 suite.return_code = 0 50 return False 51 52 if len(suite.tests) == 0: 53 logger.info("Skipping %ss, no tests to run", suite.test_kind) 54 55 # Set a successful return code on the test suite because we want to output the tests 56 # that would get run by any other suites the user specified. 57 suite.return_code = 0 58 return False 59 60 archive = None 61 if resmokelib.config.ARCHIVE_FILE: 62 archive = resmokelib.utils.archival.Archival( 63 archival_json_file=resmokelib.config.ARCHIVE_FILE, 64 limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, 65 limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, 66 logger=logger) 67 68 executor_config = suite.get_executor_config() 69 70 try: 71 executor = resmokelib.testing.executor.TestSuiteExecutor( 72 logger, suite, archive_instance=archive, **executor_config) 73 executor.run() 74 if suite.options.fail_fast and suite.return_code != 0: 75 return False 76 except (resmokelib.errors.UserInterrupt, resmokelib.errors.LoggerRuntimeConfigError) as err: 77 logger.error("Encountered an error when running %ss of suite %s: %s", 78 suite.test_kind, suite.get_display_name(), err) 79 suite.return_code = err.EXIT_CODE 80 return True 81 except: 82 logger.exception("Encountered an error when running %ss of suite %s.", 83 suite.test_kind, suite.get_display_name()) 84 suite.return_code = 2 85 return False 86 finally: 87 if archive: 88 archive.exit() 89 90 91def _log_summary(logger, suites, time_taken): 92 if len(suites) > 1: 93 resmokelib.testing.suite.Suite.log_summaries(logger, suites, time_taken) 94 95 96def _summarize_suite(suite): 97 sb = [] 98 suite.summarize(sb) 99 return "\n".join(sb) 100 101 102def _dump_suite_config(suite, logging_config): 103 """ 104 Returns a string that represents the YAML configuration of a suite. 105 106 TODO: include the "options" key in the result 107 """ 108 109 sb = [] 110 sb.append("YAML configuration of suite %s" % (suite.get_display_name())) 111 sb.append(resmokelib.utils.dump_yaml({"test_kind": suite.get_test_kind_config()})) 112 sb.append("") 113 sb.append(resmokelib.utils.dump_yaml({"selector": suite.get_selector_config()})) 114 sb.append("") 115 sb.append(resmokelib.utils.dump_yaml({"executor": suite.get_executor_config()})) 116 sb.append("") 117 sb.append(resmokelib.utils.dump_yaml({"logging": logging_config})) 118 return "\n".join(sb) 119 120 121def find_suites_by_test(suites): 122 """ 123 Looks up what other resmoke suites run the tests specified in the suites 124 parameter. Returns a dict keyed by test name, value is array of suite names. 125 """ 126 127 memberships = {} 128 test_membership = resmokelib.parser.create_test_membership_map() 129 for suite in suites: 130 for test in suite.tests: 131 memberships[test] = test_membership[test] 132 return memberships 133 134 135def _list_suites_and_exit(logger, exit_code=0): 136 suite_names = resmokelib.parser.get_named_suites() 137 logger.info("Suites available to execute:\n%s", "\n".join(suite_names)) 138 sys.exit(exit_code) 139 140 141class Main(object): 142 """ 143 A class for executing potentially multiple resmoke.py test suites. 144 """ 145 146 def __init__(self): 147 """ 148 Initializes the Main instance by parsing the command line arguments. 149 """ 150 151 self.__start_time = time.time() 152 153 values, args = resmokelib.parser.parse_command_line() 154 self.__values = values 155 self.__args = args 156 157 def _get_suites(self): 158 """ 159 Returns a list of resmokelib.testing.suite.Suite instances to execute. 160 """ 161 162 return resmokelib.parser.get_suites(self.__values, self.__args) 163 164 def run(self): 165 """ 166 Executes the list of resmokelib.testing.suite.Suite instances returned by _get_suites(). 167 """ 168 169 logging_config = resmokelib.parser.get_logging_config(self.__values) 170 resmokelib.logging.loggers.configure_loggers(logging_config) 171 resmokelib.logging.flush.start_thread() 172 173 resmokelib.parser.update_config_vars(self.__values) 174 175 exec_logger = resmokelib.logging.loggers.EXECUTOR_LOGGER 176 resmoke_logger = exec_logger.new_resmoke_logger() 177 178 if self.__values.list_suites: 179 _list_suites_and_exit(resmoke_logger) 180 181 # Log the command line arguments specified to resmoke.py to make it easier to re-run the 182 # resmoke.py invocation used by an Evergreen task. 183 resmoke_logger.info("resmoke.py invocation: %s", " ".join(sys.argv)) 184 185 interrupted = False 186 try: 187 suites = self._get_suites() 188 except resmokelib.errors.SuiteNotFound as err: 189 resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err)) 190 _list_suites_and_exit(resmoke_logger, exit_code=1) 191 192 # Register a signal handler or Windows event object so we can write the report file if the 193 # task times out. 194 resmokelib.sighandler.register(resmoke_logger, suites, self.__start_time) 195 196 # Run the suite finder after the test suite parsing is complete. 197 if self.__values.find_suites: 198 suites_by_test = find_suites_by_test(suites) 199 for test in sorted(suites_by_test): 200 suite_names = suites_by_test[test] 201 resmoke_logger.info("%s will be run by the following suite(s): %s", 202 test, suite_names) 203 sys.exit(0) 204 205 exit_code = 0 206 try: 207 for suite in suites: 208 resmoke_logger.info(_dump_suite_config(suite, logging_config)) 209 210 suite.record_suite_start() 211 interrupted = _execute_suite(suite) 212 suite.record_suite_end() 213 214 resmoke_logger.info("=" * 80) 215 resmoke_logger.info("Summary of %s suite: %s", 216 suite.get_display_name(), _summarize_suite(suite)) 217 218 if interrupted or (suite.options.fail_fast and suite.return_code != 0): 219 time_taken = time.time() - self.__start_time 220 _log_summary(resmoke_logger, suites, time_taken) 221 exit_code = suite.return_code 222 sys.exit(exit_code) 223 224 time_taken = time.time() - self.__start_time 225 _log_summary(resmoke_logger, suites, time_taken) 226 227 # Exit with a nonzero code if any of the suites failed. 228 exit_code = max(suite.return_code for suite in suites) 229 sys.exit(exit_code) 230 finally: 231 # We want to exit as quickly as possible when interrupted by a user and therefore don't 232 # bother waiting for all log output to be flushed to logkeeper. 233 # 234 # If we already failed to write log output to logkeeper, then we don't bother waiting 235 # for any remaining log output to be flushed as it'll likely fail too. Exiting without 236 # joining the flush thread here also means that resmoke.py won't hang due a logger from 237 # a fixture or a background hook not being closed. 238 if not interrupted and not resmokelib.logging.buildlogger.is_log_output_incomplete(): 239 resmokelib.logging.flush.stop_thread() 240 241 resmokelib.reportfile.write(suites) 242 243 if not interrupted and resmokelib.logging.buildlogger.is_log_output_incomplete(): 244 if exit_code == 0: 245 # We don't anticipate users to look at passing Evergreen tasks very often that 246 # even if the log output is incomplete, we'd still rather not show anything in 247 # the Evergreen UI or cause a JIRA ticket to be created. 248 resmoke_logger.info( 249 "We failed to flush all log output to logkeeper but all tests passed, so" 250 " ignoring.") 251 else: 252 resmoke_logger.info( 253 "Exiting with code %d rather than requested code %d because we failed to" 254 " flush all log output to logkeeper.", 255 resmokelib.errors.LoggerRuntimeConfigError.EXIT_CODE, exit_code) 256 sys.exit(resmokelib.errors.LoggerRuntimeConfigError.EXIT_CODE) 257 258if __name__ == "__main__": 259 Main().run() 260