1#!/usr/bin/env python
2from __future__ import print_function
3
4import re, string, sys, os, time
5
6DEBUG = 0
7testDirName = 'llvm-test'
8test      = ['compile', 'llc', 'jit', 'cbe']
9exectime     = ['llc-time', 'jit-time', 'cbe-time',]
10comptime     = ['llc', 'jit-comptime', 'compile']
11
12(tp, exp) = ('compileTime_', 'executeTime_')
13
14def parse(file):
15  f=open(file, 'r')
16  d = f.read()
17
18  #Cleanup weird stuff
19  d = re.sub(r',\d+:\d','', d)
20
21  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
22
23  test = {}
24  fname = ''
25  for t in r:
26    if DEBUG:
27      print(t)
28    if t[0] == 'PASS' or t[0] == 'FAIL' :
29      tmp = t[2].split(testDirName)
30
31      if DEBUG:
32        print(tmp)
33
34      if len(tmp) == 2:
35        fname = tmp[1].strip('\r\n')
36      else:
37        fname = tmp[0].strip('\r\n')
38
39      if fname not in test :
40        test[fname] = {}
41
42      for k in test:
43        test[fname][k] = 'NA'
44        test[fname][t[1]] = t[0]
45        if DEBUG:
46          print(test[fname][t[1]])
47    else :
48      try:
49        n = t[0].split('RESULT-')[1]
50
51        if DEBUG:
52          print(n);
53
54        if n == 'llc' or n == 'jit-comptime' or n == 'compile':
55          test[fname][tp + n] = float(t[2].split(' ')[2])
56          if DEBUG:
57            print(test[fname][tp + n])
58
59        elif n.endswith('-time') :
60            test[fname][exp + n] = float(t[2].strip('\r\n'))
61            if DEBUG:
62              print(test[fname][exp + n])
63
64        else :
65          print("ERROR!")
66          sys.exit(1)
67
68      except:
69          continue
70
71  return test
72
73# Diff results and look for regressions.
74def diffResults(d_old, d_new):
75
76  for t in sorted(d_old.keys()) :
77    if DEBUG:
78      print(t)
79
80    if t in d_new :
81
82      # Check if the test passed or failed.
83      for x in test:
84        if x in d_old[t]:
85          if x in d_new[t]:
86            if d_old[t][x] == 'PASS':
87              if d_new[t][x] != 'PASS':
88                print(t + " *** REGRESSION (" + x + ")\n")
89            else:
90              if d_new[t][x] == 'PASS':
91                print(t + " * NEW PASS (" + x + ")\n")
92
93          else :
94            print(t + "*** REGRESSION (" + x + ")\n")
95
96        # For execution time, if there is no result, its a fail.
97        for x in exectime:
98          if tp + x in d_old[t]:
99            if tp + x not in d_new[t]:
100              print(t + " *** REGRESSION (" + tp + x + ")\n")
101
102          else :
103            if tp + x in d_new[t]:
104              print(t + " * NEW PASS (" + tp + x + ")\n")
105
106
107        for x in comptime:
108          if exp + x in d_old[t]:
109            if exp + x not in d_new[t]:
110              print(t + " *** REGRESSION (" + exp + x + ")\n")
111
112          else :
113            if exp + x in d_new[t]:
114              print(t + " * NEW PASS (" + exp + x + ")\n")
115
116    else :
117      print(t + ": Removed from test-suite.\n")
118
119
120#Main
121if len(sys.argv) < 3 :
122    print('Usage:', sys.argv[0], \
123          '<old log> <new log>')
124    sys.exit(-1)
125
126d_old = parse(sys.argv[1])
127d_new = parse(sys.argv[2])
128
129
130diffResults(d_old, d_new)
131
132
133