1#! /usr/bin/env python3 2## 3## Copyright (C) by Argonne National Laboratory 4## See COPYRIGHT in top-level directory 5## 6 7import sys 8import os 9import time 10import argparse 11import subprocess 12import signal 13import datetime 14import xml.etree.ElementTree as ET 15 16class colors: 17 FAILURE = '\033[1;31m' # red 18 SUCCESS = '\033[1;32m' # green 19 INFO = '\033[1;33m' # yellow 20 PREFIX = '\033[1;36m' # cyan 21 OTHER = '\033[1;35m' # purple 22 END = '\033[0m' # reset 23 24opts = {"verbose": 0} 25 26# junit variables 27num_tests = 0 28num_failures = 0 29testnames = [] 30testtimes = [] 31testretvals = [] 32testoutputs = [] 33 34 35def init_colors(): 36 if not sys.stdout.isatty(): 37 colors.FAILURE = '' 38 colors.SUCCESS = '' 39 colors.INFO = '' 40 colors.PREFIX = '' 41 colors.OTHER = '' 42 colors.END = '' 43 44 45def printout_make(line, ret, elapsed_time, output): 46 global num_tests 47 global num_failures 48 49 sys.stdout.write(colors.PREFIX + ">>>> " + colors.END) 50 sys.stdout.write("return status: ") 51 if (ret == 0): 52 sys.stdout.write(colors.SUCCESS + "SUCCESS\n" + colors.END) 53 else: 54 sys.stdout.write(colors.FAILURE + "FAILURE\n" + colors.END) 55 sys.stdout.write(colors.PREFIX + ">>>> " + colors.END) 56 sys.stdout.write("elapsed time: %f sec\n" % elapsed_time) 57 if (ret != 0): 58 num_tests = num_tests + 1 59 num_failures = num_failures + 1 60 if (output != ""): 61 execname = line.split(' ', 1)[0].rstrip() 62 print(colors.FAILURE + "\n==== \"make %s\" failed ====" % execname + colors.END) 63 print(output) 64 print(colors.FAILURE + "==== make output complete ====\n" + colors.END) 65 testnames.append(line) 66 testtimes.append(elapsed_time) 67 testretvals.append(ret) 68 testoutputs.append(output) 69 70 71def printout_exec(line, ret, elapsed_time, output): 72 global num_tests 73 global num_failures 74 75 num_tests = num_tests + 1 76 sys.stdout.write(colors.PREFIX + ">>>> " + colors.END) 77 sys.stdout.write("return status: ") 78 if (ret == 0): 79 sys.stdout.write(colors.SUCCESS + "SUCCESS\n" + colors.END) 80 else: 81 sys.stdout.write(colors.FAILURE + "FAILURE\n" + colors.END) 82 num_failures = num_failures + 1 83 sys.stdout.write(colors.PREFIX + ">>>> " + colors.END) 84 sys.stdout.write("elapsed time: %f sec\n" % elapsed_time) 85 if (ret != 0): 86 if (output != ""): 87 print(colors.FAILURE + "==== execution failed with the following output ====" + colors.END) 88 print(output) 89 print(colors.FAILURE + "==== execution output complete ====\n" + colors.END) 90 91 testnames.append(line) 92 testtimes.append(elapsed_time) 93 testretvals.append(ret) 94 testoutputs.append(output) 95 96 97def getlines(fh): 98 alllines = fh.readlines() 99 reallines = [] 100 for line in alllines: 101 # skip comments 102 if line.startswith("#"): 103 continue 104 # skip empty lines 105 if not line.strip(): 106 continue 107 reallines.append(line) 108 return reallines 109 110 111def create_summary(summary_file): 112 global num_tests 113 global num_failures 114 115 # open the summary file and write to it 116 try: 117 fh = open(summary_file, "w") 118 except: 119 sys.stderr.write(colors.FAILURE + ">>>> ERROR: " + colors.END) 120 sys.stderr.write("could not open summary file %s\n" % summary_file) 121 sys.exit() 122 fh.write("<testsuites>\n") 123 fh.write(" <testsuite failures=\"%d\"\n" % num_failures) 124 fh.write(" errors=\"0\"\n") 125 fh.write(" skipped=\"0\"\n") 126 fh.write(" tests=\"%d\"\n" % num_tests) 127 fh.write(" date=\"%s\"\n" % datetime.datetime.now()) 128 fh.write(" name=\"summary_junit_xml\">\n") 129 130 for x in range(len(testnames)): 131 fh.write(" <testcase name=\"%s\" time=\"%f\">\n" % (testnames[x].strip(), testtimes[x])) 132 if (testretvals[x] != 0): 133 fh.write(" <failure><![CDATA[\n") 134 if (testoutputs[x]): 135 fh.write(testoutputs[x] + "\n") 136 else: 137 fh.write("test failed\n") 138 fh.write(" ]]></failure>\n") 139 fh.write(" </testcase>\n") 140 141 fh.write(" </testsuite>\n") 142 fh.write("</testsuites>\n") 143 fh.close() 144 145 146def wait_with_signal(p): 147 try: 148 ret = p.wait() 149 except: 150 p.kill() 151 p.wait() 152 sys.exit() 153 return ret 154 155 156def run_testlist(testlist): 157 try: 158 fh = open(testlist, "r") 159 except: 160 sys.stderr.write(colors.FAILURE + ">>>> ERROR: " + colors.END) 161 sys.stderr.write("could not open testlist %s\n" % testlist) 162 sys.exit() 163 164 print(colors.INFO + "\n==== executing testlist %s ====" % testlist + colors.END) 165 166 lines = getlines(fh) 167 168 firstline = 1 169 for line in lines: 170 if (firstline): 171 firstline = 0 172 else: 173 print("") 174 175 176 ############################################################################ 177 # if the first argument is a directory, step into the 178 # directory and reexecute make 179 ############################################################################ 180 dirname = line.split(' ', 1)[0].rstrip() 181 if (os.path.isdir(dirname)): 182 sys.stdout.write(colors.PREFIX + ">>>> " + colors.END) 183 sys.stdout.write(colors.OTHER + "stepping into directory %s\n" % dirname + colors.END) 184 olddir = os.getcwd() 185 os.chdir(dirname) 186 chdirargs = "make -s testing".split(' ') 187 chdirargs = map(lambda s: s.strip(), chdirargs) 188 p = subprocess.Popen(chdirargs) 189 wait_with_signal(p) 190 os.chdir(olddir) 191 continue 192 193 194 # command line to process 195 sys.stdout.write(line) 196 197 198 ############################################################################ 199 # make executable 200 ############################################################################ 201 execname = line.split(' ', 1)[0].rstrip() 202 if opts['verbose']: 203 print("make " + execname) 204 start = time.time() 205 p = subprocess.Popen(['make', execname], stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 206 ret = wait_with_signal(p) 207 out = p.communicate() 208 end = time.time() 209 if opts['verbose']: 210 print(out[0].decode().strip()) 211 printout_make(line, ret, end - start, out[0].decode().strip()) 212 if (ret != 0): 213 continue # skip over to the next line 214 215 216 ############################################################################ 217 # run the executable 218 ############################################################################ 219 fullcmd = "./" + line 220 cmdargs = fullcmd.split(' ') 221 cmdargs = map(lambda s: s.strip(), cmdargs) 222 start = time.time() 223 p = subprocess.Popen(cmdargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 224 ret = wait_with_signal(p) 225 out = p.communicate() 226 end = time.time() 227 printout_exec(line, ret, end - start, out[0].decode().strip()) 228 229 fh.close() 230 print(colors.INFO + "==== done executing testlist %s ====" % testlist + colors.END) 231 232 233if __name__ == '__main__': 234 init_colors() 235 236 parser = argparse.ArgumentParser() 237 parser.add_argument('testlists', help='testlist files to execute', nargs='+') 238 parser.add_argument('--summary', help='file to write the summary to', required=True) 239 args = parser.parse_args() 240 241 if os.environ.get('V'): 242 opts["verbose"] = 1 243 244 for testlist in args.testlists: 245 run_testlist(os.path.abspath(testlist)) 246 create_summary(os.path.abspath(args.summary)) 247 248 249