1#!/usr/bin/python 2# Copyright (c) 2011 The Native Client Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Simple test suite for toolchains espcially llvm arm toolchains. 7 8Sample invocations 9 10tools/toolchain_tester/toolchain_tester.py [options]+ test1.c test2.c ... 11 12where options are 13 14--config <config> 15--append <tag>=<value> 16--append_file=<filename> 17--verbose 18--show_console 19--exclude=<filename> 20--tmp=<path> 21--check_excludes 22--concurrency=<number> 23 24e.g. --append "CFLAGS:-lsupc++" will enable C++ eh support 25 26NOTE: the location of tmp files is intentionally hardcoded, so you 27can only run one instance of this at a time. 28""" 29 30from __future__ import print_function 31 32import getopt 33import glob 34import multiprocessing 35import os 36import shlex 37import subprocess 38import sys 39import time 40 41import toolchain_config 42# ====================================================================== 43# Options 44# ====================================================================== 45# list of streams being logged to (both normal and verbose output) 46REPORT_STREAMS = [sys.stdout] 47# max time (secs) to wait for command any command to complete 48TIMEOUT = 120 49# enable verbose output, e.g. commands being executed 50VERBOSE = 0 51# prefix for temporary files 52TMP_PREFIX = '/tmp/tc_test_' 53# show command output (stdout/stderr) 54SHOW_CONSOLE = 1 55# append these settings to config 56APPEND = [] 57# append these settings to config, for a given test (specified by APPEND_FILES) 58APPEND_PER_TEST = {} 59# Files listing the APPEND_PER_TEST entries. 60APPEND_FILES = [] 61# exclude these tests 62EXCLUDE = {} 63# check whether excludes are still necessary 64CHECK_EXCLUDES = 0 65# Files listing excluded tests 66EXCLUDE_FILES = [] 67# module with settings for compiler, etc. 68CFG = None 69# Number of simultaneous test processes 70CONCURRENCY = 1 71# Child processes push failing test results onto this queue 72ERRORS = multiprocessing.Queue() 73# ====================================================================== 74# Hook print to we can print to both stdout and a file 75def Print(message): 76 for s in REPORT_STREAMS: 77 print(message, file=s) 78 79 80# ====================================================================== 81def Banner(message): 82 Print('=' * 70) 83 Print(message) 84 Print('=' * 70) 85 86# ====================================================================== 87def RunCommand(cmd, always_dump_stdout_stderr): 88 """Run a shell command given as an argv style vector.""" 89 if VERBOSE: 90 Print(str(cmd)) 91 Print(" ".join(cmd)) 92 start = time.time() 93 p = subprocess.Popen(cmd, 94 bufsize=1000*1000, 95 stderr=subprocess.PIPE, 96 stdout=subprocess.PIPE) 97 while p.poll() is None: 98 time.sleep(0.1) 99 now = time.time() 100 if now - start > TIMEOUT: 101 Print('Error: timeout') 102 Print('Killing pid %d' % p.pid) 103 os.waitpid(-1, os.WNOHANG) 104 return -1 105 stdout = p.stdout.read() 106 stderr = p.stderr.read() 107 retcode = p.wait() 108 109 if retcode != 0: 110 Print('Error: command failed %d %s' % (retcode, ' '.join(cmd))) 111 always_dump_stdout_stderr = True 112 113 if always_dump_stdout_stderr: 114 Print(stderr) 115 Print(stdout) 116 return retcode 117 118 119def RemoveTempFiles(): 120 global TMP_PREFIX 121 for f in glob.glob(TMP_PREFIX + '*'): 122 os.remove(f) 123 124 125def MakeExecutableCustom(config, test, extra): 126 global TMP_PREFIX 127 global SHOW_CONSOLE 128 d = extra.copy() 129 d['tmp'] = (TMP_PREFIX + '_' + 130 os.path.basename(os.path.dirname(test)) + '_' + 131 os.path.basename(test)) 132 d['src'] = test 133 for phase, command in config.GetCommands(d): 134 command = shlex.split(command) 135 try: 136 retcode = RunCommand(command, SHOW_CONSOLE) 137 except Exception, err: 138 Print("cannot run phase %s: %s" % (phase, str(err))) 139 return phase 140 if retcode: 141 return phase 142 # success 143 return '' 144 145 146def ParseExcludeFiles(config_attributes): 147 ''' Parse the files containing tests to exclude (i.e. expected fails). 148 Each line may contain a comma-separated list of attributes restricting 149 the test configurations which are expected to fail. (e.g. architecture 150 or optimization level). A test is only excluded if the configuration 151 has all the attributes specified in the exclude line. Lines which 152 have no attributes will match everything, and lines which specify only 153 one attribute (e.g. architecture) will match all configurations with that 154 attributed (e.g. both opt levels with that architecture) 155 ''' 156 for excludefile in EXCLUDE_FILES: 157 f = open(excludefile) 158 for line in f: 159 line = line.strip() 160 if not line: continue 161 if line.startswith('#'): continue 162 tokens = line.split() 163 if len(tokens) > 1: 164 attributes = set(tokens[1].split(',')) 165 if not attributes.issubset(config_attributes): 166 continue 167 test = tokens[0] 168 else: 169 test = line 170 if test in EXCLUDE: 171 Print('ERROR: duplicate exclude: [%s]' % line) 172 EXCLUDE[test] = excludefile 173 f.close() 174 Print('Size of excludes now: %d' % len(EXCLUDE)) 175 176 177def ParseAppendFiles(): 178 """Parse the file contain a list of test + CFLAGS to append for that test.""" 179 for append_file in APPEND_FILES: 180 f = open(append_file) 181 for line in f: 182 line = line.strip() 183 if not line: continue 184 if line.startswith('#'): continue 185 tokens = line.split(',') 186 test = tokens[0] 187 to_append = {} 188 for t in tokens[1:]: 189 tag, value = t.split(':') 190 if tag in to_append: 191 to_append[tag] = to_append[tag] + ' ' + value 192 else: 193 to_append[tag] = value 194 if test in APPEND_PER_TEST: 195 raise Exception('Duplicate append/flags for test %s (old %s, new %s)' % 196 (test, APPEND_PER_TEST[test], to_append)) 197 APPEND_PER_TEST[test] = to_append 198 f.close() 199 200 201def ParseCommandLineArgs(argv): 202 """Process command line options and return the unprocessed left overs.""" 203 global VERBOSE, COMPILE_MODE, RUN_MODE, TMP_PREFIX 204 global CFG, APPEND, SHOW_CONSOLE, CHECK_EXCLUDES, CONCURRENCY 205 try: 206 opts, args = getopt.getopt(argv[1:], '', 207 ['verbose', 208 'show_console', 209 'append=', 210 'append_file=', 211 'config=', 212 'exclude=', 213 'check_excludes', 214 'tmp=', 215 'concurrency=']) 216 except getopt.GetoptError, err: 217 Print(str(err)) # will print something like 'option -a not recognized' 218 sys.exit(-1) 219 220 for o, a in opts: 221 # strip the leading '--' 222 o = o[2:] 223 if o == 'verbose': 224 VERBOSE = 1 225 elif o == 'show_console': 226 SHOW_CONSOLE = 1 227 elif o == 'check_excludes': 228 CHECK_EXCLUDES = 1 229 elif o == 'tmp': 230 TMP_PREFIX = a 231 elif o == 'exclude': 232 # Parsing of exclude files must happen after we know the current config 233 EXCLUDE_FILES.append(a) 234 elif o == 'append': 235 tag, value = a.split(":", 1) 236 APPEND.append((tag, value)) 237 elif o == 'append_file': 238 APPEND_FILES.append(a) 239 elif o == 'config': 240 CFG = a 241 elif o == 'concurrency': 242 CONCURRENCY = int(a) 243 else: 244 Print('ERROR: bad commandline arg: %s' % o) 245 sys.exit(-1) 246 # return the unprocessed options, i.e. the command 247 return args 248 249 250def RunTest(args): 251 num, total, config, test, extra_flags = args 252 base_test_name = os.path.basename(test) 253 extra_flags = extra_flags.copy() 254 toolchain_config.AppendDictionary(extra_flags, 255 APPEND_PER_TEST.get(base_test_name, {})) 256 Print('Running %d/%d: %s' % (num + 1, total, base_test_name)) 257 try: 258 result = MakeExecutableCustom(config, test, extra_flags) 259 except KeyboardInterrupt: 260 # Swallow the keyboard interrupt in the child. Otherwise the parent 261 # hangs trying to join it. 262 pass 263 264 if result and config.IsFlaky(): 265 # TODO(dschuff): deflake qemu or switch to hardware 266 # BUG=http://code.google.com/p/nativeclient/issues/detail?id=2197 267 # try it again, and only fail on consecutive failures 268 Print('Retrying ' + base_test_name) 269 result = MakeExecutableCustom(config, test, extra_flags) 270 if result: 271 Print('[ FAILED ] %s: %s' % (result, test)) 272 ERRORS.put((result, test)) 273 274 275def RunSuite(config, files, extra_flags, errors): 276 """Run a collection of benchmarks.""" 277 global ERRORS, CONCURRENCY 278 Banner('running %d tests' % (len(files))) 279 pool = multiprocessing.Pool(processes=CONCURRENCY) 280 # create a list of run arguments to map over 281 argslist = [(num, len(files), config, test, extra_flags) 282 for num, test in enumerate(files)] 283 # let the process pool handle the test assignments, order doesn't matter 284 pool.map(RunTest, argslist) 285 while not ERRORS.empty(): 286 phase, test = ERRORS.get() 287 errors[phase].append(test) 288 289 290def FilterOutExcludedTests(files, exclude): 291 return [f for f in files if not os.path.basename(f) in exclude] 292 293 294def main(argv): 295 files = ParseCommandLineArgs(argv) 296 297 if not CFG: 298 print('ERROR: you must specify a toolchain-config using --config=<config>') 299 print('Available configs are: ') 300 print('\n'.join(toolchain_config.TOOLCHAIN_CONFIGS.keys())) 301 print() 302 return -1 303 304 global TMP_PREFIX 305 global APPEND 306 TMP_PREFIX = TMP_PREFIX + CFG 307 308 Banner('Config: %s' % CFG) 309 config = toolchain_config.TOOLCHAIN_CONFIGS[CFG] 310 ParseExcludeFiles(config.GetAttributes()) 311 for tag, value in APPEND: 312 config.Append(tag, value) 313 ParseAppendFiles() 314 config.SanityCheck() 315 Print('TMP_PREFIX: %s' % TMP_PREFIX) 316 317 # initialize error stats 318 errors = {} 319 for phase in config.GetPhases(): 320 errors[phase] = [] 321 322 Print('Tests before filtering %d' % len(files)) 323 if not CHECK_EXCLUDES: 324 files = FilterOutExcludedTests(files, EXCLUDE) 325 Print('Tests after filtering %d' % len(files)) 326 try: 327 RunSuite(config, files, {}, errors) 328 finally: 329 RemoveTempFiles() 330 331 # print error report 332 USED_EXCLUDES = {} 333 num_errors = 0 334 for k in errors: 335 lst = errors[k] 336 if not lst: continue 337 Banner('%d failures in config %s phase %s' % (len(lst), CFG, k)) 338 for e in lst: 339 if os.path.basename(e) in EXCLUDE: 340 USED_EXCLUDES[os.path.basename(e)] = None 341 continue 342 Print(e) 343 num_errors += 1 344 345 if CHECK_EXCLUDES: 346 Banner('Unnecessary excludes:') 347 for e in EXCLUDE: 348 if e not in USED_EXCLUDES: 349 Print(e + ' (' + EXCLUDE[e] + ')') 350 return num_errors > 0 351 352if __name__ == '__main__': 353 sys.exit(main(sys.argv)) 354