1#!/usr/bin/env python3
2
3import argparse
4import glob
5import json
6import os
7import re
8import xml.dom.minidom as md
9
10
11TEST_COLLECTIONS = {
12    "EUnit": "src/**/.eunit/*.xml",
13    "EXUnit": "_build/integration/lib/couchdbtest/*.xml",
14    "Mango": "src/mango/*.xml",
15    "JavaScript": "test/javascript/*.xml",
16}
17
18
19def _attrs(elem):
20    ret = {}
21    for (k, v) in elem.attributes.items():
22        ret[k.lower()] = v
23    return ret
24
25
26def _text(elem):
27    rc = []
28    for node in elem.childNodes:
29        if node.nodeType == node.TEXT_NODE:
30            rc.append(node.data)
31        else:
32            rc.append(self._text(node))
33    return "".join(rc)
34
35
36class TestCase(object):
37    def __init__(self, elem):
38        self.elem = elem
39
40        attrs = _attrs(elem)
41
42        self.name = self._name(attrs)
43        self.time = float(attrs["time"])
44
45        self.failure = False
46        self._check_failure(elem, attrs)
47
48        self.error = False
49        self._check_error(elem, attrs)
50
51        self.skipped = False
52        self._check_skipped(elem, attrs)
53
54    def _check_failure(self, elem, attrs):
55        failures = elem.getElementsByTagName("failure")
56        if not failures:
57            return
58
59        self.failure = True
60        self.failure_msg = _text(failures[0]).strip()
61
62    def _check_error(self, elem, attrs):
63        errors = elem.getElementsByTagName("error")
64        if not errors:
65            return
66
67        self.error = True
68        self.error_msg = _text(errors[0]).strip()
69
70    def _check_skipped(self, elem, attrs):
71        skipped = elem.getElementsByTagName("skipped")
72        if not skipped:
73            return
74
75        attrs = _attrs(skipped[0])
76        self.skipped = True
77        self.skipped_msg = attrs.get("message", attrs.get("type", "<unknown>"))
78
79    def _name(self, attrs):
80        klass = attrs.get("classname", "")
81        if klass.startswith("Elixir."):
82            klass = klass[len("Elixir.") :]
83        if klass:
84            return "%s - %s" % (klass, attrs["name"])
85        return attrs["name"]
86
87
88class TestSuite(object):
89    SUITE_NAME_PATTERNS = [re.compile("module '([^']+)'"), re.compile("Elixir\.(.+)")]
90
91    def __init__(self, elem):
92        self.elem = elem
93
94        attrs = _attrs(elem)
95
96        self.name = self._name(attrs)
97
98        self.time = 0.0
99        if "time" in attrs:
100            self.time = float(attrs["time"])
101
102        self.num_tests = int(attrs["tests"])
103        self.num_failures = int(attrs["failures"])
104        self.num_errors = int(attrs["errors"])
105        self.num_skipped = 0
106
107        self.tests = []
108        self.test_time = 0.0
109
110        for t_elem in elem.getElementsByTagName("testcase"):
111            self.tests.append(TestCase(t_elem))
112            self.test_time += self.tests[-1].time
113            if self.tests[-1].skipped:
114                self.num_skipped += 1
115
116        if self.time == 0.0 and self.test_time > 0.0:
117            self.time = self.test_time
118
119    def _name(self, attrs):
120        raw_name = attrs["name"]
121        for p in self.SUITE_NAME_PATTERNS:
122            match = p.match(raw_name)
123            if match:
124                return match.group(1)
125        return raw_name
126
127
128class TestCollection(object):
129    def __init__(self, name, pattern):
130        self.name = name
131        self.pattern = pattern
132        self.suites = []
133        self.bad_files = []
134
135        for fname in glob.glob(pattern):
136            self._load_file(fname)
137
138    def _load_file(self, filename):
139        try:
140            dom = md.parse(filename)
141        except:
142            self.bad_files.append(filename)
143            return
144        for elem in dom.getElementsByTagName("testsuite"):
145            self.suites.append(TestSuite(elem))
146
147
148def parse_args():
149    parser = argparse.ArgumentParser(description="Show test result summaries")
150    parser.add_argument(
151        "--ignore-failures",
152        action="store_true",
153        default=False,
154        help="Don't display test failures",
155    )
156    parser.add_argument(
157        "--ignore-errors",
158        action="store_true",
159        default=False,
160        help="Don't display test errors",
161    )
162    parser.add_argument(
163        "--ignore-skipped",
164        action="store_true",
165        default=False,
166        help="Don't display skipped tests",
167    )
168    parser.add_argument(
169        "--all", type=int, default=0, help="Number of rows to show for all groups"
170    )
171    parser.add_argument(
172        "--collection",
173        action="append",
174        default=[],
175        help="Which collection to display. May be repeated.",
176    )
177    parser.add_argument(
178        "--suites", type=int, default=0, help="Number of suites to show"
179    )
180    parser.add_argument("--tests", type=int, default=0, help="Number of tests to show")
181    parser.add_argument(
182        "--sort",
183        default="total",
184        choices=["test", "fixture", "total"],
185        help="Timing column to sort on",
186    )
187    return parser.parse_args()
188
189
190def display_failures(collections):
191    failures = []
192    for collection in collections:
193        for suite in collection.suites:
194            for test in suite.tests:
195                if not test.failure:
196                    continue
197                failures.append((test.name, test.failure_msg))
198
199    if not len(failures):
200        return
201    print("Failures")
202    print("========")
203    print()
204    for failure in failures:
205        print(failure[0])
206        print("-" * len(failure[0]))
207        print()
208        print(failure[1])
209        print()
210
211
212def display_errors(collections):
213    errors = []
214    for collection in collections:
215        for suite in collection.suites:
216            for test in suite.tests:
217                if not test.error:
218                    continue
219                errors.append((test.name, test.error_msg))
220
221    if not len(errors):
222        return
223    print("Errors")
224    print("======")
225    print()
226    for error in errors:
227        print(error[0])
228        print("-" * len(error[0]))
229        print()
230        print(error[1])
231        print()
232
233
234def display_skipped(collections):
235    skipped = []
236    for collection in collections:
237        for suite in collection.suites:
238            for test in suite.tests:
239                if not test.skipped:
240                    continue
241                name = "%s - %s - %s" % (collection.name, suite.name, test.name)
242                skipped.append((name, test.skipped_msg))
243    if not skipped:
244        return
245    print("Skipped")
246    print("=======")
247    print()
248    for row in sorted(skipped):
249        print("  %s: %s" % row)
250    print()
251
252
253def display_table(table):
254    for ridx, row in enumerate(table):
255        new_row = []
256        for col in row:
257            if isinstance(col, float):
258                new_row.append("%4.1fs" % col)
259            elif isinstance(col, int):
260                new_row.append("%d" % col)
261            else:
262                new_row.append(col)
263        table[ridx] = new_row
264    for row in table:
265        fmt = " ".join(["%10s"] * len(row))
266        print(fmt % tuple(row))
267
268
269def display_collections(collections, sort):
270    rows = []
271    for collection in collections:
272        total_time = 0.0
273        test_time = 0.0
274        num_tests = 0
275        num_failures = 0
276        num_errors = 0
277        num_skipped = 0
278        for suite in collection.suites:
279            total_time += suite.time
280            test_time += suite.test_time
281            num_tests += suite.num_tests
282            num_failures += suite.num_failures
283            num_errors += suite.num_errors
284            num_skipped += suite.num_skipped
285        cols = (
286            total_time,
287            max(0.0, total_time - test_time),
288            test_time,
289            num_tests,
290            num_failures,
291            num_errors,
292            num_skipped,
293            collection.name + "        ",
294        )
295        rows.append(cols)
296
297    scol = 0
298    if sort == "fixture":
299        scol = 1
300    elif sort == "test":
301        scol = 2
302
303    def skey(row):
304        return (-1.0 * row[scol], row[-1])
305
306    rows.sort(key=skey)
307
308    print("Collections")
309    print("===========")
310    print()
311    headers = ["Total", "Fixture", "Test", "Count", "Failed", "Errors", "Skipped"]
312    display_table([headers] + rows)
313    print()
314
315
316def display_suites(collections, count, sort):
317    rows = []
318    for collection in collections:
319        for suite in collection.suites:
320            cols = [
321                suite.time,
322                max(0.0, suite.time - suite.test_time),
323                suite.test_time,
324                suite.num_tests,
325                suite.num_failures,
326                suite.num_errors,
327                suite.num_skipped,
328                collection.name + " - " + suite.name,
329            ]
330            rows.append(cols)
331
332    scol = 0
333    if sort == "fixture":
334        scol = 1
335    elif sort == "test":
336        scol = 2
337
338    def skey(row):
339        return (-1.0 * row[scol], row[-1])
340
341    rows.sort(key=skey)
342
343    rows = rows[:count]
344
345    print("Suites")
346    print("======")
347    print()
348    headers = ["Total", "Fixture", "Test", "Count", "Failed", "Errors", "Skipped"]
349    display_table([headers] + rows)
350    print()
351
352
353def display_tests(collections, count):
354    rows = []
355    for collection in collections:
356        for suite in collection.suites:
357            for test in suite.tests:
358                if test.failure or test.error or test.skipped:
359                    continue
360                fmt = "%s - %s - %s"
361                display = fmt % (collection.name, suite.name, test.name)
362                rows.append((test.time, display))
363
364    def skey(row):
365        return (-1.0 * row[0], row[-1])
366
367    rows.sort(key=skey)
368    rows = rows[:count]
369
370    print("Tests")
371    print("=====")
372    print()
373    display_table(rows)
374    print()
375
376
377def main():
378    args = parse_args()
379
380    if not args.collection:
381        args.collection = ["eunit", "exunit", "mango", "javascript"]
382
383    collections = []
384    for (name, pattern) in TEST_COLLECTIONS.items():
385        if name.lower() not in args.collection:
386            continue
387        collections.append(TestCollection(name, pattern))
388
389    if not args.ignore_failures:
390        display_failures(collections)
391
392    if not args.ignore_errors:
393        display_errors(collections)
394
395    if not args.ignore_skipped:
396        display_skipped(collections)
397
398    display_collections(collections, args.sort)
399
400    if args.all > 0:
401        args.suites = args.all
402        args.tests = args.all
403
404    if args.suites > 0:
405        display_suites(collections, args.suites, args.sort)
406
407    if args.tests > 0:
408        display_tests(collections, args.tests)
409
410
411if __name__ == "__main__":
412    main()
413