1#!/usr/bin/env python
2
3from __future__ import print_function
4import re, string, sys, os, time, math
5
6DEBUG = 0
7
8(tp, exp) = ('compile', 'exec')
9
10def parse(file):
11  f = open(file, 'r')
12  d = f.read()
13
14  # Cleanup weird stuff
15  d = re.sub(r',\d+:\d', '', d)
16
17  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
18
19  test = {}
20  fname = ''
21  for t in r:
22    if DEBUG:
23      print(t)
24
25    if t[0] == 'PASS' or t[0] == 'FAIL' :
26      tmp = t[2].split('llvm-test/')
27
28      if DEBUG:
29        print(tmp)
30
31      if len(tmp) == 2:
32        fname = tmp[1].strip('\r\n')
33      else:
34        fname = tmp[0].strip('\r\n')
35
36      if fname not in test:
37        test[fname] = {}
38
39      test[fname][t[1] + ' state'] = t[0]
40      test[fname][t[1] + ' time'] = float('nan')
41    else :
42      try:
43        n = t[0].split('RESULT-')[1]
44
45        if DEBUG:
46          print("n == ", n);
47
48        if n == 'compile-success':
49          test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
50
51        elif n == 'exec-success':
52          test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
53          if DEBUG:
54            print(test[fname][string.replace(n, '-success', '')])
55
56        else :
57          # print "ERROR!"
58          sys.exit(1)
59
60      except:
61          continue
62
63  return test
64
65# Diff results and look for regressions.
66def diffResults(d_old, d_new):
67  regressions = {}
68  passes = {}
69  removed = ''
70
71  for x in ['compile state', 'compile time', 'exec state', 'exec time']:
72    regressions[x] = ''
73    passes[x] = ''
74
75  for t in sorted(d_old.keys()) :
76    if t in d_new:
77
78      # Check if the test passed or failed.
79      for x in ['compile state', 'compile time', 'exec state', 'exec time']:
80
81        if x not in d_old[t] and x not in d_new[t]:
82          continue
83
84        if x in d_old[t]:
85          if x in d_new[t]:
86
87            if d_old[t][x] == 'PASS':
88              if d_new[t][x] != 'PASS':
89                regressions[x] += t + "\n"
90            else:
91              if d_new[t][x] == 'PASS':
92                passes[x] += t + "\n"
93
94          else :
95            regressions[x] += t + "\n"
96
97        if x == 'compile state' or x == 'exec state':
98          continue
99
100        # For execution time, if there is no result it's a fail.
101        if x not in d_old[t] and x not in d_new[t]:
102          continue
103        elif x not in d_new[t]:
104          regressions[x] += t + "\n"
105        elif x not in d_old[t]:
106          passes[x] += t + "\n"
107
108        if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
109          continue
110
111        elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
112          passes[x] += t + "\n"
113
114        elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
115          regressions[x] += t + ": NaN%\n"
116
117        if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
118              (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
119          regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"
120
121    else :
122      removed += t + "\n"
123
124  if len(regressions['compile state']) != 0:
125    print('REGRESSION: Compilation Failed')
126    print(regressions['compile state'])
127
128  if len(regressions['exec state']) != 0:
129    print('REGRESSION: Execution Failed')
130    print(regressions['exec state'])
131
132  if len(regressions['compile time']) != 0:
133    print('REGRESSION: Compilation Time')
134    print(regressions['compile time'])
135
136  if len(regressions['exec time']) != 0:
137    print('REGRESSION: Execution Time')
138    print(regressions['exec time'])
139
140  if len(passes['compile state']) != 0:
141    print('NEW PASSES: Compilation')
142    print(passes['compile state'])
143
144  if len(passes['exec state']) != 0:
145    print('NEW PASSES: Execution')
146    print(passes['exec state'])
147
148  if len(removed) != 0:
149    print('REMOVED TESTS')
150    print(removed)
151
152# Main
153if len(sys.argv) < 3 :
154  print('Usage:', sys.argv[0], '<old log> <new log>')
155  sys.exit(-1)
156
157d_old = parse(sys.argv[1])
158d_new = parse(sys.argv[2])
159
160diffResults(d_old, d_new)
161