1#!/usr/bin/env python3 2 3import argparse 4import glob 5import json 6import os 7import re 8import xml.dom.minidom as md 9 10 11TEST_COLLECTIONS = { 12 "EUnit": "src/**/.eunit/*.xml", 13 "EXUnit": "_build/integration/lib/couchdbtest/*.xml", 14 "Mango": "src/mango/*.xml", 15 "JavaScript": "test/javascript/*.xml", 16} 17 18 19def _attrs(elem): 20 ret = {} 21 for (k, v) in elem.attributes.items(): 22 ret[k.lower()] = v 23 return ret 24 25 26def _text(elem): 27 rc = [] 28 for node in elem.childNodes: 29 if node.nodeType == node.TEXT_NODE: 30 rc.append(node.data) 31 else: 32 rc.append(self._text(node)) 33 return "".join(rc) 34 35 36class TestCase(object): 37 def __init__(self, elem): 38 self.elem = elem 39 40 attrs = _attrs(elem) 41 42 self.name = self._name(attrs) 43 self.time = float(attrs["time"]) 44 45 self.failure = False 46 self._check_failure(elem, attrs) 47 48 self.error = False 49 self._check_error(elem, attrs) 50 51 self.skipped = False 52 self._check_skipped(elem, attrs) 53 54 def _check_failure(self, elem, attrs): 55 failures = elem.getElementsByTagName("failure") 56 if not failures: 57 return 58 59 self.failure = True 60 self.failure_msg = _text(failures[0]).strip() 61 62 def _check_error(self, elem, attrs): 63 errors = elem.getElementsByTagName("error") 64 if not errors: 65 return 66 67 self.error = True 68 self.error_msg = _text(errors[0]).strip() 69 70 def _check_skipped(self, elem, attrs): 71 skipped = elem.getElementsByTagName("skipped") 72 if not skipped: 73 return 74 75 attrs = _attrs(skipped[0]) 76 self.skipped = True 77 self.skipped_msg = attrs.get("message", attrs.get("type", "<unknown>")) 78 79 def _name(self, attrs): 80 klass = attrs.get("classname", "") 81 if klass.startswith("Elixir."): 82 klass = klass[len("Elixir.") :] 83 if klass: 84 return "%s - %s" % (klass, attrs["name"]) 85 return attrs["name"] 86 87 88class TestSuite(object): 89 SUITE_NAME_PATTERNS = [re.compile("module '([^']+)'"), re.compile("Elixir\.(.+)")] 90 91 def __init__(self, elem): 92 self.elem = elem 93 94 attrs = _attrs(elem) 95 96 self.name = self._name(attrs) 97 98 self.time = 0.0 99 if "time" in attrs: 100 self.time = float(attrs["time"]) 101 102 self.num_tests = int(attrs["tests"]) 103 self.num_failures = int(attrs["failures"]) 104 self.num_errors = int(attrs["errors"]) 105 self.num_skipped = 0 106 107 self.tests = [] 108 self.test_time = 0.0 109 110 for t_elem in elem.getElementsByTagName("testcase"): 111 self.tests.append(TestCase(t_elem)) 112 self.test_time += self.tests[-1].time 113 if self.tests[-1].skipped: 114 self.num_skipped += 1 115 116 if self.time == 0.0 and self.test_time > 0.0: 117 self.time = self.test_time 118 119 def _name(self, attrs): 120 raw_name = attrs["name"] 121 for p in self.SUITE_NAME_PATTERNS: 122 match = p.match(raw_name) 123 if match: 124 return match.group(1) 125 return raw_name 126 127 128class TestCollection(object): 129 def __init__(self, name, pattern): 130 self.name = name 131 self.pattern = pattern 132 self.suites = [] 133 self.bad_files = [] 134 135 for fname in glob.glob(pattern): 136 self._load_file(fname) 137 138 def _load_file(self, filename): 139 try: 140 dom = md.parse(filename) 141 except: 142 self.bad_files.append(filename) 143 return 144 for elem in dom.getElementsByTagName("testsuite"): 145 self.suites.append(TestSuite(elem)) 146 147 148def parse_args(): 149 parser = argparse.ArgumentParser(description="Show test result summaries") 150 parser.add_argument( 151 "--ignore-failures", 152 action="store_true", 153 default=False, 154 help="Don't display test failures", 155 ) 156 parser.add_argument( 157 "--ignore-errors", 158 action="store_true", 159 default=False, 160 help="Don't display test errors", 161 ) 162 parser.add_argument( 163 "--ignore-skipped", 164 action="store_true", 165 default=False, 166 help="Don't display skipped tests", 167 ) 168 parser.add_argument( 169 "--all", type=int, default=0, help="Number of rows to show for all groups" 170 ) 171 parser.add_argument( 172 "--collection", 173 action="append", 174 default=[], 175 help="Which collection to display. May be repeated.", 176 ) 177 parser.add_argument( 178 "--suites", type=int, default=0, help="Number of suites to show" 179 ) 180 parser.add_argument("--tests", type=int, default=0, help="Number of tests to show") 181 parser.add_argument( 182 "--sort", 183 default="total", 184 choices=["test", "fixture", "total"], 185 help="Timing column to sort on", 186 ) 187 return parser.parse_args() 188 189 190def display_failures(collections): 191 failures = [] 192 for collection in collections: 193 for suite in collection.suites: 194 for test in suite.tests: 195 if not test.failure: 196 continue 197 failures.append((test.name, test.failure_msg)) 198 199 if not len(failures): 200 return 201 print("Failures") 202 print("========") 203 print() 204 for failure in failures: 205 print(failure[0]) 206 print("-" * len(failure[0])) 207 print() 208 print(failure[1]) 209 print() 210 211 212def display_errors(collections): 213 errors = [] 214 for collection in collections: 215 for suite in collection.suites: 216 for test in suite.tests: 217 if not test.error: 218 continue 219 errors.append((test.name, test.error_msg)) 220 221 if not len(errors): 222 return 223 print("Errors") 224 print("======") 225 print() 226 for error in errors: 227 print(error[0]) 228 print("-" * len(error[0])) 229 print() 230 print(error[1]) 231 print() 232 233 234def display_skipped(collections): 235 skipped = [] 236 for collection in collections: 237 for suite in collection.suites: 238 for test in suite.tests: 239 if not test.skipped: 240 continue 241 name = "%s - %s - %s" % (collection.name, suite.name, test.name) 242 skipped.append((name, test.skipped_msg)) 243 if not skipped: 244 return 245 print("Skipped") 246 print("=======") 247 print() 248 for row in sorted(skipped): 249 print(" %s: %s" % row) 250 print() 251 252 253def display_table(table): 254 for ridx, row in enumerate(table): 255 new_row = [] 256 for col in row: 257 if isinstance(col, float): 258 new_row.append("%4.1fs" % col) 259 elif isinstance(col, int): 260 new_row.append("%d" % col) 261 else: 262 new_row.append(col) 263 table[ridx] = new_row 264 for row in table: 265 fmt = " ".join(["%10s"] * len(row)) 266 print(fmt % tuple(row)) 267 268 269def display_collections(collections, sort): 270 rows = [] 271 for collection in collections: 272 total_time = 0.0 273 test_time = 0.0 274 num_tests = 0 275 num_failures = 0 276 num_errors = 0 277 num_skipped = 0 278 for suite in collection.suites: 279 total_time += suite.time 280 test_time += suite.test_time 281 num_tests += suite.num_tests 282 num_failures += suite.num_failures 283 num_errors += suite.num_errors 284 num_skipped += suite.num_skipped 285 cols = ( 286 total_time, 287 max(0.0, total_time - test_time), 288 test_time, 289 num_tests, 290 num_failures, 291 num_errors, 292 num_skipped, 293 collection.name + " ", 294 ) 295 rows.append(cols) 296 297 scol = 0 298 if sort == "fixture": 299 scol = 1 300 elif sort == "test": 301 scol = 2 302 303 def skey(row): 304 return (-1.0 * row[scol], row[-1]) 305 306 rows.sort(key=skey) 307 308 print("Collections") 309 print("===========") 310 print() 311 headers = ["Total", "Fixture", "Test", "Count", "Failed", "Errors", "Skipped"] 312 display_table([headers] + rows) 313 print() 314 315 316def display_suites(collections, count, sort): 317 rows = [] 318 for collection in collections: 319 for suite in collection.suites: 320 cols = [ 321 suite.time, 322 max(0.0, suite.time - suite.test_time), 323 suite.test_time, 324 suite.num_tests, 325 suite.num_failures, 326 suite.num_errors, 327 suite.num_skipped, 328 collection.name + " - " + suite.name, 329 ] 330 rows.append(cols) 331 332 scol = 0 333 if sort == "fixture": 334 scol = 1 335 elif sort == "test": 336 scol = 2 337 338 def skey(row): 339 return (-1.0 * row[scol], row[-1]) 340 341 rows.sort(key=skey) 342 343 rows = rows[:count] 344 345 print("Suites") 346 print("======") 347 print() 348 headers = ["Total", "Fixture", "Test", "Count", "Failed", "Errors", "Skipped"] 349 display_table([headers] + rows) 350 print() 351 352 353def display_tests(collections, count): 354 rows = [] 355 for collection in collections: 356 for suite in collection.suites: 357 for test in suite.tests: 358 if test.failure or test.error or test.skipped: 359 continue 360 fmt = "%s - %s - %s" 361 display = fmt % (collection.name, suite.name, test.name) 362 rows.append((test.time, display)) 363 364 def skey(row): 365 return (-1.0 * row[0], row[-1]) 366 367 rows.sort(key=skey) 368 rows = rows[:count] 369 370 print("Tests") 371 print("=====") 372 print() 373 display_table(rows) 374 print() 375 376 377def main(): 378 args = parse_args() 379 380 if not args.collection: 381 args.collection = ["eunit", "exunit", "mango", "javascript"] 382 383 collections = [] 384 for (name, pattern) in TEST_COLLECTIONS.items(): 385 if name.lower() not in args.collection: 386 continue 387 collections.append(TestCollection(name, pattern)) 388 389 if not args.ignore_failures: 390 display_failures(collections) 391 392 if not args.ignore_errors: 393 display_errors(collections) 394 395 if not args.ignore_skipped: 396 display_skipped(collections) 397 398 display_collections(collections, args.sort) 399 400 if args.all > 0: 401 args.suites = args.all 402 args.tests = args.all 403 404 if args.suites > 0: 405 display_suites(collections, args.suites, args.sort) 406 407 if args.tests > 0: 408 display_tests(collections, args.tests) 409 410 411if __name__ == "__main__": 412 main() 413