1""" 2Holder for the (test kind, list of tests) pair with additional metadata about when and how they 3execute. 4""" 5 6from __future__ import absolute_import 7 8import itertools 9import threading 10import time 11 12from . import report as _report 13from . import summary as _summary 14from .. import config as _config 15from .. import selector as _selector 16 17 18def synchronized(method): 19 """Decorator to enfore instance lock ownership when calling the method.""" 20 21 def synced(self, *args, **kwargs): 22 lock = getattr(self, "_lock") 23 with lock: 24 return method(self, *args, **kwargs) 25 26 return synced 27 28 29class Suite(object): 30 """ 31 A suite of tests of a particular kind (e.g. C++ unit tests, dbtests, jstests). 32 """ 33 34 def __init__(self, suite_name, suite_config, suite_options=_config.SuiteOptions.ALL_INHERITED): 35 """ 36 Initializes the suite with the specified name and configuration. 37 """ 38 self._lock = threading.RLock() 39 40 self._suite_name = suite_name 41 self._suite_config = suite_config 42 self._suite_options = suite_options 43 44 self.test_kind = self.get_test_kind_config() 45 self.tests = self._get_tests_for_kind(self.test_kind) 46 47 self.return_code = None # Set by the executor. 48 49 self._suite_start_time = None 50 self._suite_end_time = None 51 52 self._test_start_times = [] 53 self._test_end_times = [] 54 self._reports = [] 55 56 # We keep a reference to the TestReports from the currently running jobs so that we can 57 # report intermediate results. 58 self._partial_reports = None 59 60 def _get_tests_for_kind(self, test_kind): 61 """ 62 Returns the tests to run based on the 'test_kind'-specific 63 filtering policy. 64 """ 65 test_info = self.get_selector_config() 66 67 # The mongos_test doesn't have to filter anything, the test_info is just the arguments to 68 # the mongos program to be used as the test case. 69 if test_kind == "mongos_test": 70 mongos_options = test_info # Just for easier reading. 71 if not isinstance(mongos_options, dict): 72 raise TypeError("Expected dictionary of arguments to mongos") 73 return [mongos_options] 74 75 tests = _selector.filter_tests(test_kind, test_info) 76 if _config.ORDER_TESTS_BY_NAME: 77 return sorted(tests, key=str.lower) 78 79 return tests 80 81 def get_name(self): 82 """ 83 Returns the name of the test suite. 84 """ 85 return self._suite_name 86 87 def get_display_name(self): 88 """ 89 Returns the name of the test suite with a unique identifier for its SuiteOptions. 90 """ 91 92 if self.options.description is None: 93 return self.get_name() 94 95 return "{} ({})".format(self.get_name(), self.options.description) 96 97 def get_selector_config(self): 98 """ 99 Returns the "selector" section of the YAML configuration. 100 """ 101 102 selector = self._suite_config["selector"].copy() 103 104 if self.options.include_tags is not None: 105 if "include_tags" in selector: 106 selector["include_tags"] = {"$allOf": [ 107 selector["include_tags"], 108 self.options.include_tags, 109 ]} 110 elif "exclude_tags" in selector: 111 selector["exclude_tags"] = {"$anyOf": [ 112 selector["exclude_tags"], 113 {"$not": self.options.include_tags}, 114 ]} 115 else: 116 selector["include_tags"] = self.options.include_tags 117 118 return selector 119 120 def get_executor_config(self): 121 """ 122 Returns the "executor" section of the YAML configuration. 123 """ 124 return self._suite_config["executor"] 125 126 def get_test_kind_config(self): 127 """ 128 Returns the "test_kind" section of the YAML configuration. 129 """ 130 return self._suite_config["test_kind"] 131 132 @property 133 def options(self): 134 return self._suite_options.resolve() 135 136 def with_options(self, suite_options): 137 """ 138 Returns a Suite instance with the specified resmokelib.config.SuiteOptions. 139 """ 140 141 return Suite(self._suite_name, self._suite_config, suite_options) 142 143 @synchronized 144 def record_suite_start(self): 145 """ 146 Records the start time of the suite. 147 """ 148 self._suite_start_time = time.time() 149 150 @synchronized 151 def record_suite_end(self): 152 """ 153 Records the end time of the suite. 154 """ 155 self._suite_end_time = time.time() 156 157 @synchronized 158 def record_test_start(self, partial_reports): 159 """ 160 Records the start time of an execution and stores the 161 TestReports for currently running jobs. 162 """ 163 self._test_start_times.append(time.time()) 164 self._partial_reports = partial_reports 165 166 @synchronized 167 def record_test_end(self, report): 168 """ 169 Records the end time of an execution. 170 """ 171 self._test_end_times.append(time.time()) 172 self._reports.append(report) 173 self._partial_reports = None 174 175 @synchronized 176 def get_active_report(self): 177 """ 178 Returns the partial report of the currently running execution, if there is one. 179 """ 180 if not self._partial_reports: 181 return None 182 return _report.TestReport.combine(*self._partial_reports) 183 184 @synchronized 185 def get_reports(self): 186 """ 187 Returns the list of reports. If there's an execution currently 188 in progress, then a report for the partial results is included 189 in the returned list. 190 """ 191 192 if self._partial_reports is not None: 193 return self._reports + [self.get_active_report()] 194 195 return self._reports 196 197 @synchronized 198 def summarize(self, sb): 199 """ 200 Appends a summary of the suite onto the string builder 'sb'. 201 """ 202 if not self._reports and not self._partial_reports: 203 sb.append("No tests ran.") 204 summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) 205 elif not self._reports and self._partial_reports: 206 summary = self.summarize_latest(sb) 207 elif len(self._reports) == 1 and not self._partial_reports: 208 summary = self._summarize_execution(0, sb) 209 else: 210 summary = self._summarize_repeated(sb) 211 212 summarized_group = " %ss: %s" % (self.test_kind, "\n ".join(sb)) 213 214 if summary.num_run == 0: 215 sb.append("Suite did not run any tests.") 216 return 217 218 # Override the 'time_taken' attribute of the summary if we have more accurate timing 219 # information available. 220 if self._suite_start_time is not None and self._suite_end_time is not None: 221 time_taken = self._suite_end_time - self._suite_start_time 222 summary = summary._replace(time_taken=time_taken) 223 224 sb.append("%d test(s) ran in %0.2f seconds" 225 " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary) 226 227 sb.append(summarized_group) 228 229 @synchronized 230 def summarize_latest(self, sb): 231 """ 232 Returns a summary of the latest execution of the suite and appends a 233 summary of that execution onto the string builder 'sb'. 234 235 If there's an execution currently in progress, then the partial 236 summary of that execution is appended to 'sb'. 237 """ 238 239 if self._partial_reports is None: 240 return self._summarize_execution(-1, sb) 241 242 active_report = _report.TestReport.combine(*self._partial_reports) 243 # Use the current time as the time that this suite finished running. 244 end_time = time.time() 245 return self._summarize_report(active_report, self._test_start_times[-1], end_time, sb) 246 247 def _summarize_repeated(self, sb): 248 """ 249 Returns the summary information of all executions and appends 250 each execution's summary onto the string builder 'sb'. Also 251 appends information of how many repetitions there were. 252 """ 253 254 reports = self.get_reports() # Also includes the combined partial reports. 255 num_iterations = len(reports) 256 start_times = self._test_start_times[:] 257 end_times = self._test_end_times[:] 258 if self._partial_reports: 259 end_times.append(time.time()) # Add an end time in this copy for the partial reports. 260 261 total_time_taken = end_times[-1] - start_times[0] 262 sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken)) 263 264 combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) 265 for iteration in xrange(num_iterations): 266 # Summarize each execution as a bulleted list of results. 267 bulleter_sb = [] 268 summary = self._summarize_report( 269 reports[iteration], 270 start_times[iteration], 271 end_times[iteration], 272 bulleter_sb) 273 combined_summary = _summary.combine(combined_summary, summary) 274 275 for (i, line) in enumerate(bulleter_sb): 276 # Only bullet first line, indent others. 277 prefix = "* " if i == 0 else " " 278 sb.append(prefix + line) 279 280 return combined_summary 281 282 def _summarize_execution(self, iteration, sb): 283 """ 284 Returns the summary information of the execution given by 285 'iteration' and appends a summary of that execution onto the 286 string builder 'sb'. 287 """ 288 289 return self._summarize_report(self._reports[iteration], 290 self._test_start_times[iteration], 291 self._test_end_times[iteration], 292 sb) 293 294 def _summarize_report(self, report, start_time, end_time, sb): 295 """ 296 Returns the summary information of the execution given by 297 'report' that started at 'start_time' and finished at 298 'end_time', and appends a summary of that execution onto the 299 string builder 'sb'. 300 """ 301 302 time_taken = end_time - start_time 303 304 # Tests that were interrupted are treated as failures because (1) the test has already been 305 # started and therefore isn't skipped and (2) the test has yet to finish and therefore 306 # cannot be said to have succeeded. 307 num_failed = report.num_failed + report.num_interrupted 308 num_run = report.num_succeeded + report.num_errored + num_failed 309 num_skipped = len(self.tests) + report.num_dynamic - num_run 310 311 if report.num_succeeded == num_run and num_skipped == 0: 312 sb.append("All %d test(s) passed in %0.2f seconds." % (num_run, time_taken)) 313 return _summary.Summary(num_run, time_taken, num_run, 0, 0, 0) 314 315 summary = _summary.Summary(num_run, time_taken, report.num_succeeded, num_skipped, 316 num_failed, report.num_errored) 317 318 sb.append("%d test(s) ran in %0.2f seconds" 319 " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary) 320 321 if num_failed > 0: 322 sb.append("The following tests failed (with exit code):") 323 for test_info in itertools.chain(report.get_failed(), report.get_interrupted()): 324 sb.append(" %s (%d)" % (test_info.test_id, test_info.return_code)) 325 326 if report.num_errored > 0: 327 sb.append("The following tests had errors:") 328 for test_info in report.get_errored(): 329 sb.append(" %s" % (test_info.test_id)) 330 331 return summary 332 333 @staticmethod 334 def log_summaries(logger, suites, time_taken): 335 sb = [] 336 sb.append("Summary of all suites: %d suites ran in %0.2f seconds" 337 % (len(suites), time_taken)) 338 for suite in suites: 339 suite_sb = [] 340 suite.summarize(suite_sb) 341 sb.append(" %s: %s" % (suite.get_display_name(), "\n ".join(suite_sb))) 342 343 logger.info("=" * 80) 344 logger.info("\n".join(sb)) 345