1#!/usr/bin/python
2# Copyright (c) 2011 The Native Client Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Simple test suite for toolchains espcially llvm arm toolchains.
7
8Sample invocations
9
10tools/toolchain_tester/toolchain_tester.py  [options]+ test1.c test2.c ...
11
12where options are
13
14--config <config>
15--append <tag>=<value>
16--append_file=<filename>
17--verbose
18--show_console
19--exclude=<filename>
20--tmp=<path>
21--check_excludes
22--concurrency=<number>
23
24e.g. --append "CFLAGS:-lsupc++" will enable C++ eh support
25
26NOTE: the location of tmp files is intentionally hardcoded, so you
27can only run one instance of this at a time.
28"""
29
30from __future__ import print_function
31
32import getopt
33import glob
34import multiprocessing
35import os
36import shlex
37import subprocess
38import sys
39import time
40
41import toolchain_config
42# ======================================================================
43# Options
44# ======================================================================
45# list of streams being logged to (both normal and verbose output)
46REPORT_STREAMS = [sys.stdout]
47# max time (secs) to wait for command any command to complete
48TIMEOUT = 120
49# enable verbose output, e.g. commands being executed
50VERBOSE = 0
51# prefix for temporary files
52TMP_PREFIX = '/tmp/tc_test_'
53# show command output (stdout/stderr)
54SHOW_CONSOLE = 1
55# append these settings to config
56APPEND = []
57# append these settings to config, for a given test (specified by APPEND_FILES)
58APPEND_PER_TEST = {}
59# Files listing the APPEND_PER_TEST entries.
60APPEND_FILES = []
61# exclude these tests
62EXCLUDE = {}
63# check whether excludes are still necessary
64CHECK_EXCLUDES = 0
65# Files listing excluded tests
66EXCLUDE_FILES = []
67# module with settings for compiler, etc.
68CFG = None
69# Number of simultaneous test processes
70CONCURRENCY = 1
71# Child processes push failing test results onto this queue
72ERRORS = multiprocessing.Queue()
73# ======================================================================
74# Hook print to we can print to both stdout and a file
75def Print(message):
76  for s in REPORT_STREAMS:
77    print(message, file=s)
78
79
80# ======================================================================
81def Banner(message):
82  Print('=' * 70)
83  Print(message)
84  Print('=' * 70)
85
86# ======================================================================
87def RunCommand(cmd, always_dump_stdout_stderr):
88  """Run a shell command given as an argv style vector."""
89  if VERBOSE:
90    Print(str(cmd))
91    Print(" ".join(cmd))
92  start = time.time()
93  p = subprocess.Popen(cmd,
94                       bufsize=1000*1000,
95                       stderr=subprocess.PIPE,
96                       stdout=subprocess.PIPE)
97  while p.poll() is None:
98    time.sleep(0.1)
99    now = time.time()
100    if now - start > TIMEOUT:
101      Print('Error: timeout')
102      Print('Killing pid %d' % p.pid)
103      os.waitpid(-1, os.WNOHANG)
104      return -1
105  stdout = p.stdout.read()
106  stderr = p.stderr.read()
107  retcode = p.wait()
108
109  if retcode != 0:
110    Print('Error: command failed %d %s' % (retcode, ' '.join(cmd)))
111    always_dump_stdout_stderr = True
112
113  if always_dump_stdout_stderr:
114    Print(stderr)
115    Print(stdout)
116  return retcode
117
118
119def RemoveTempFiles():
120  global TMP_PREFIX
121  for f in glob.glob(TMP_PREFIX + '*'):
122    os.remove(f)
123
124
125def MakeExecutableCustom(config, test, extra):
126  global TMP_PREFIX
127  global SHOW_CONSOLE
128  d = extra.copy()
129  d['tmp'] = (TMP_PREFIX + '_' +
130              os.path.basename(os.path.dirname(test)) + '_' +
131              os.path.basename(test))
132  d['src'] = test
133  for phase, command in config.GetCommands(d):
134    command = shlex.split(command)
135    try:
136      retcode = RunCommand(command, SHOW_CONSOLE)
137    except Exception, err:
138      Print("cannot run phase %s: %s" % (phase, str(err)))
139      return phase
140    if retcode:
141      return phase
142  # success
143  return ''
144
145
146def ParseExcludeFiles(config_attributes):
147  ''' Parse the files containing tests to exclude (i.e. expected fails).
148  Each line may contain a comma-separated list of attributes restricting
149  the test configurations which are expected to fail. (e.g. architecture
150  or optimization level). A test is only excluded if the configuration
151  has all the attributes specified in the exclude line. Lines which
152  have no attributes will match everything, and lines which specify only
153  one attribute (e.g. architecture) will match all configurations with that
154  attributed (e.g. both opt levels with that architecture)
155  '''
156  for excludefile in EXCLUDE_FILES:
157    f = open(excludefile)
158    for line in f:
159      line = line.strip()
160      if not line: continue
161      if line.startswith('#'): continue
162      tokens = line.split()
163      if len(tokens) > 1:
164        attributes = set(tokens[1].split(','))
165        if not attributes.issubset(config_attributes):
166          continue
167        test = tokens[0]
168      else:
169        test = line
170      if test in EXCLUDE:
171        Print('ERROR: duplicate exclude: [%s]' % line)
172      EXCLUDE[test] = excludefile
173    f.close()
174    Print('Size of excludes now: %d' % len(EXCLUDE))
175
176
177def ParseAppendFiles():
178  """Parse the file contain a list of test + CFLAGS to append for that test."""
179  for append_file in APPEND_FILES:
180    f = open(append_file)
181    for line in f:
182      line = line.strip()
183      if not line: continue
184      if line.startswith('#'): continue
185      tokens = line.split(',')
186      test = tokens[0]
187      to_append = {}
188      for t in tokens[1:]:
189        tag, value = t.split(':')
190        if tag in to_append:
191          to_append[tag] = to_append[tag] + ' ' + value
192        else:
193          to_append[tag] = value
194      if test in APPEND_PER_TEST:
195        raise Exception('Duplicate append/flags for test %s (old %s, new %s)' %
196                        (test, APPEND_PER_TEST[test], to_append))
197      APPEND_PER_TEST[test] = to_append
198    f.close()
199
200
201def ParseCommandLineArgs(argv):
202  """Process command line options and return the unprocessed left overs."""
203  global VERBOSE, COMPILE_MODE, RUN_MODE, TMP_PREFIX
204  global CFG, APPEND, SHOW_CONSOLE, CHECK_EXCLUDES, CONCURRENCY
205  try:
206    opts, args = getopt.getopt(argv[1:], '',
207                               ['verbose',
208                                'show_console',
209                                'append=',
210                                'append_file=',
211                                'config=',
212                                'exclude=',
213                                'check_excludes',
214                                'tmp=',
215                                'concurrency='])
216  except getopt.GetoptError, err:
217    Print(str(err))  # will print something like 'option -a not recognized'
218    sys.exit(-1)
219
220  for o, a in opts:
221    # strip the leading '--'
222    o = o[2:]
223    if o == 'verbose':
224      VERBOSE = 1
225    elif o == 'show_console':
226      SHOW_CONSOLE = 1
227    elif o == 'check_excludes':
228      CHECK_EXCLUDES = 1
229    elif o == 'tmp':
230      TMP_PREFIX = a
231    elif o == 'exclude':
232      # Parsing of exclude files must happen after we know the current config
233      EXCLUDE_FILES.append(a)
234    elif o == 'append':
235      tag, value = a.split(":", 1)
236      APPEND.append((tag, value))
237    elif o == 'append_file':
238      APPEND_FILES.append(a)
239    elif o == 'config':
240      CFG = a
241    elif o == 'concurrency':
242      CONCURRENCY = int(a)
243    else:
244      Print('ERROR: bad commandline arg: %s' % o)
245      sys.exit(-1)
246    # return the unprocessed options, i.e. the command
247  return args
248
249
250def RunTest(args):
251  num, total, config, test, extra_flags = args
252  base_test_name = os.path.basename(test)
253  extra_flags = extra_flags.copy()
254  toolchain_config.AppendDictionary(extra_flags,
255                                    APPEND_PER_TEST.get(base_test_name, {}))
256  Print('Running %d/%d: %s' % (num + 1, total, base_test_name))
257  try:
258    result = MakeExecutableCustom(config, test, extra_flags)
259  except KeyboardInterrupt:
260    # Swallow the keyboard interrupt in the child. Otherwise the parent
261    # hangs trying to join it.
262    pass
263
264  if result and config.IsFlaky():
265    # TODO(dschuff): deflake qemu or switch to hardware
266    # BUG=http://code.google.com/p/nativeclient/issues/detail?id=2197
267    # try it again, and only fail on consecutive failures
268    Print('Retrying ' + base_test_name)
269    result = MakeExecutableCustom(config, test, extra_flags)
270  if result:
271    Print('[  FAILED  ] %s: %s' % (result, test))
272    ERRORS.put((result, test))
273
274
275def RunSuite(config, files, extra_flags, errors):
276  """Run a collection of benchmarks."""
277  global ERRORS, CONCURRENCY
278  Banner('running %d tests' % (len(files)))
279  pool = multiprocessing.Pool(processes=CONCURRENCY)
280  # create a list of run arguments to map over
281  argslist = [(num, len(files), config, test, extra_flags)
282         for num, test in enumerate(files)]
283  # let the process pool handle the test assignments, order doesn't matter
284  pool.map(RunTest, argslist)
285  while not ERRORS.empty():
286    phase, test = ERRORS.get()
287    errors[phase].append(test)
288
289
290def FilterOutExcludedTests(files, exclude):
291  return  [f for f in files if not os.path.basename(f) in exclude]
292
293
294def main(argv):
295  files = ParseCommandLineArgs(argv)
296
297  if not CFG:
298    print('ERROR: you must specify a toolchain-config using --config=<config>')
299    print('Available configs are: ')
300    print('\n'.join(toolchain_config.TOOLCHAIN_CONFIGS.keys()))
301    print()
302    return -1
303
304  global TMP_PREFIX
305  global APPEND
306  TMP_PREFIX = TMP_PREFIX + CFG
307
308  Banner('Config: %s' % CFG)
309  config = toolchain_config.TOOLCHAIN_CONFIGS[CFG]
310  ParseExcludeFiles(config.GetAttributes())
311  for tag, value in APPEND:
312    config.Append(tag, value)
313  ParseAppendFiles()
314  config.SanityCheck()
315  Print('TMP_PREFIX: %s' % TMP_PREFIX)
316
317  # initialize error stats
318  errors = {}
319  for phase in config.GetPhases():
320    errors[phase] = []
321
322  Print('Tests before filtering %d' % len(files))
323  if not CHECK_EXCLUDES:
324    files = FilterOutExcludedTests(files, EXCLUDE)
325  Print('Tests after filtering %d' % len(files))
326  try:
327    RunSuite(config, files, {}, errors)
328  finally:
329    RemoveTempFiles()
330
331  # print error report
332  USED_EXCLUDES = {}
333  num_errors = 0
334  for k in errors:
335    lst = errors[k]
336    if not lst: continue
337    Banner('%d failures in config %s phase %s' % (len(lst), CFG, k))
338    for e in lst:
339      if os.path.basename(e) in EXCLUDE:
340        USED_EXCLUDES[os.path.basename(e)] = None
341        continue
342      Print(e)
343      num_errors += 1
344
345  if CHECK_EXCLUDES:
346    Banner('Unnecessary excludes:')
347    for e in EXCLUDE:
348      if e not in USED_EXCLUDES:
349        Print(e + ' (' + EXCLUDE[e] + ')')
350  return num_errors > 0
351
352if __name__ == '__main__':
353  sys.exit(main(sys.argv))
354