1#! /usr/bin/env python3
2##
3## Copyright (C) by Argonne National Laboratory
4##     See COPYRIGHT in top-level directory
5##
6
7import sys
8import os
9import time
10import argparse
11import subprocess
12import signal
13import datetime
14import xml.etree.ElementTree as ET
15
16class colors:
17    FAILURE = '\033[1;31m' # red
18    SUCCESS = '\033[1;32m' # green
19    INFO    = '\033[1;33m' # yellow
20    PREFIX  = '\033[1;36m' # cyan
21    OTHER   = '\033[1;35m' # purple
22    END     = '\033[0m'    # reset
23
24opts = {"verbose": 0}
25
26# junit variables
27num_tests = 0
28num_failures = 0
29testnames = []
30testtimes = []
31testretvals = []
32testoutputs = []
33
34
35def init_colors():
36    if not sys.stdout.isatty():
37        colors.FAILURE = ''
38        colors.SUCCESS = ''
39        colors.INFO = ''
40        colors.PREFIX = ''
41        colors.OTHER = ''
42        colors.END = ''
43
44
45def printout_make(line, ret, elapsed_time, output):
46    global num_tests
47    global num_failures
48
49    sys.stdout.write(colors.PREFIX + ">>>> " + colors.END)
50    sys.stdout.write("return status: ")
51    if (ret == 0):
52        sys.stdout.write(colors.SUCCESS + "SUCCESS\n" + colors.END)
53    else:
54        sys.stdout.write(colors.FAILURE + "FAILURE\n" + colors.END)
55    sys.stdout.write(colors.PREFIX + ">>>> " + colors.END)
56    sys.stdout.write("elapsed time: %f sec\n" % elapsed_time)
57    if (ret != 0):
58        num_tests = num_tests + 1
59        num_failures = num_failures + 1
60        if (output != ""):
61            execname = line.split(' ', 1)[0].rstrip()
62            print(colors.FAILURE + "\n==== \"make %s\" failed ====" % execname + colors.END)
63            print(output)
64            print(colors.FAILURE + "==== make output complete ====\n" + colors.END)
65        testnames.append(line)
66        testtimes.append(elapsed_time)
67        testretvals.append(ret)
68        testoutputs.append(output)
69
70
71def printout_exec(line, ret, elapsed_time, output):
72    global num_tests
73    global num_failures
74
75    num_tests = num_tests + 1
76    sys.stdout.write(colors.PREFIX + ">>>> " + colors.END)
77    sys.stdout.write("return status: ")
78    if (ret == 0):
79        sys.stdout.write(colors.SUCCESS + "SUCCESS\n" + colors.END)
80    else:
81        sys.stdout.write(colors.FAILURE + "FAILURE\n" + colors.END)
82        num_failures = num_failures + 1
83    sys.stdout.write(colors.PREFIX + ">>>> " + colors.END)
84    sys.stdout.write("elapsed time: %f sec\n" % elapsed_time)
85    if (ret != 0):
86        if (output != ""):
87            print(colors.FAILURE + "==== execution failed with the following output ====" + colors.END)
88            print(output)
89            print(colors.FAILURE + "==== execution output complete ====\n" + colors.END)
90
91    testnames.append(line)
92    testtimes.append(elapsed_time)
93    testretvals.append(ret)
94    testoutputs.append(output)
95
96
97def getlines(fh):
98    alllines = fh.readlines()
99    reallines = []
100    for line in alllines:
101        # skip comments
102        if line.startswith("#"):
103            continue
104        # skip empty lines
105        if not line.strip():
106            continue
107        reallines.append(line)
108    return reallines
109
110
111def create_summary(summary_file):
112    global num_tests
113    global num_failures
114
115    # open the summary file and write to it
116    try:
117        fh = open(summary_file, "w")
118    except:
119        sys.stderr.write(colors.FAILURE + ">>>> ERROR: " + colors.END)
120        sys.stderr.write("could not open summary file %s\n" % summary_file)
121        sys.exit()
122    fh.write("<testsuites>\n")
123    fh.write("  <testsuite failures=\"%d\"\n" % num_failures)
124    fh.write("             errors=\"0\"\n")
125    fh.write("             skipped=\"0\"\n")
126    fh.write("             tests=\"%d\"\n" % num_tests)
127    fh.write("             date=\"%s\"\n" % datetime.datetime.now())
128    fh.write("             name=\"summary_junit_xml\">\n")
129
130    for x in range(len(testnames)):
131        fh.write("    <testcase name=\"%s\" time=\"%f\">\n" % (testnames[x].strip(), testtimes[x]))
132        if (testretvals[x] != 0):
133            fh.write("      <failure><![CDATA[\n")
134            if (testoutputs[x]):
135                fh.write(testoutputs[x] + "\n")
136            else:
137                fh.write("test failed\n")
138            fh.write("      ]]></failure>\n")
139        fh.write("    </testcase>\n")
140
141    fh.write("  </testsuite>\n")
142    fh.write("</testsuites>\n")
143    fh.close()
144
145
146def wait_with_signal(p):
147    try:
148        ret = p.wait()
149    except:
150        p.kill()
151        p.wait()
152        sys.exit()
153    return ret
154
155
156def run_testlist(testlist):
157    try:
158        fh = open(testlist, "r")
159    except:
160        sys.stderr.write(colors.FAILURE + ">>>> ERROR: " + colors.END)
161        sys.stderr.write("could not open testlist %s\n" % testlist)
162        sys.exit()
163
164    print(colors.INFO + "\n==== executing testlist %s ====" % testlist + colors.END)
165
166    lines = getlines(fh)
167
168    firstline = 1
169    for line in lines:
170        if (firstline):
171            firstline = 0
172        else:
173            print("")
174
175
176        ############################################################################
177        # if the first argument is a directory, step into the
178        # directory and reexecute make
179        ############################################################################
180        dirname = line.split(' ', 1)[0].rstrip()
181        if (os.path.isdir(dirname)):
182            sys.stdout.write(colors.PREFIX + ">>>> " + colors.END)
183            sys.stdout.write(colors.OTHER + "stepping into directory %s\n" % dirname + colors.END)
184            olddir = os.getcwd()
185            os.chdir(dirname)
186            chdirargs = "make -s testing".split(' ')
187            chdirargs = map(lambda s: s.strip(), chdirargs)
188            p = subprocess.Popen(chdirargs)
189            wait_with_signal(p)
190            os.chdir(olddir)
191            continue
192
193
194        # command line to process
195        sys.stdout.write(line)
196
197
198        ############################################################################
199        # make executable
200        ############################################################################
201        execname = line.split(' ', 1)[0].rstrip()
202        if opts['verbose']:
203            print("make " + execname)
204        start = time.time()
205        p = subprocess.Popen(['make', execname], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
206        ret = wait_with_signal(p)
207        out = p.communicate()
208        end = time.time()
209        if opts['verbose']:
210            print(out[0].decode().strip())
211        printout_make(line, ret, end - start, out[0].decode().strip())
212        if (ret != 0):
213            continue  # skip over to the next line
214
215
216        ############################################################################
217        # run the executable
218        ############################################################################
219        fullcmd = "./" + line
220        cmdargs = fullcmd.split(' ')
221        cmdargs = map(lambda s: s.strip(), cmdargs)
222        start = time.time()
223        p = subprocess.Popen(cmdargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
224        ret = wait_with_signal(p)
225        out = p.communicate()
226        end = time.time()
227        printout_exec(line, ret, end - start, out[0].decode().strip())
228
229    fh.close()
230    print(colors.INFO + "==== done executing testlist %s ====" % testlist + colors.END)
231
232
233if __name__ == '__main__':
234    init_colors()
235
236    parser = argparse.ArgumentParser()
237    parser.add_argument('testlists', help='testlist files to execute', nargs='+')
238    parser.add_argument('--summary', help='file to write the summary to', required=True)
239    args = parser.parse_args()
240
241    if os.environ.get('V'):
242        opts["verbose"] = 1
243
244    for testlist in args.testlists:
245        run_testlist(os.path.abspath(testlist))
246    create_summary(os.path.abspath(args.summary))
247
248
249