1#!/bin/sh
2# This script automatically test the given tool with the tool's test cases,
3# reporting anything of interest.
4
5# Written by Mike Stump <mrs@cygnus.com>
6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
7
8usage()
9{
10	if [ -n "$1" ] ; then
11		echo "$0: Error: $1" >&2
12		echo >&2
13	fi
14	cat >&2 <<EOUSAGE
15Usage: $0 [-strict] PREVIOUS CURRENT
16
17Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
18
19	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
20
21	Unless -strict is given, these discrepancies are not counted as errors:
22		missing/extra .sum files when comparing directories
23		tests that failed in PREVIOUS but pass in CURRENT
24		tests that were not in PREVIOUS but appear in CURRENT
25		tests in PREVIOUS that are missing in CURRENT
26
27	Exit with the following values:
28		0 if there is nothing of interest
29		1 if there are errors when comparing single test case files
30		N for the number of errors found when comparing directories
31EOUSAGE
32	exit 2
33}
34
35export LC_ALL=C
36
37tool=gxx
38
39TMPDIR=${TMPDIR:-/tmp}
40tmp1=$TMPDIR/$tool-testing.$$a
41tmp2=$TMPDIR/$tool-testing.$$b
42now_s=$TMPDIR/$tool-testing.$$d
43before_s=$TMPDIR/$tool-testing.$$e
44lst1=$TMPDIR/$tool-lst1.$$
45lst2=$TMPDIR/$tool-lst2.$$
46lst3=$TMPDIR/$tool-lst3.$$
47lst4=$TMPDIR/$tool-lst4.$$
48lst5=$TMPDIR/$tool-lst5.$$
49sum1=$TMPDIR/$tool-sum1.$$
50sum2=$TMPDIR/$tool-sum2.$$
51tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
52
53[ "$1" = "-strict" ] && strict=$1 && shift
54[ "$1" = "-?" ] && usage
55[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
56
57trap "rm -f $tmps" 0 1 2 3 5 9 13 15
58exit_status=0
59
60if [ -d "$1" -a -d "$2" ] ; then
61	find "$1/" -name '*.sum' >$lst1
62	find "$2/" -name '*.sum' >$lst2
63	echo "# Comparing directories"
64	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
65	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
66	echo
67	# remove leading directory components to compare
68	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
69	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
70	comm -23 $lst3 $lst4 >$lst5
71	if [ -s $lst5 ] ; then
72		echo "# Extra sum files in Dir1=$1"
73		sed -e "s|^|< $1/|" $lst5
74		echo
75		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
76	fi
77	comm -13 $lst3 $lst4 >$lst5
78	if [ -s $lst5 ] ; then
79		echo "# Extra sum files in Dir2=$2"
80		sed -e "s|^|> $2/|" $lst5
81		echo
82		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
83	fi
84	comm -12 $lst3 $lst4 | sort -u >$lst5
85	if [ ! -s $lst5 ] ; then
86		echo "# No common sum files"
87		exit_status=`expr $exit_status + 1`
88		exit $exit_status
89	fi
90	cmnsums=`cat $lst5 | wc -l`
91	echo "# Comparing $cmnsums common sum files"
92	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
93	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
94	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
95	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
96	ret=$?
97	if [ $ret -ne 0 ]; then
98		exit_status=`expr $exit_status + 1`
99		echo "## Differences found: $fname"
100	fi
101	if [ $exit_status -ne 0 ]; then
102		echo "# $exit_status differences in $cmnsums common sum files found"
103	else
104		echo "# No differences found in $cmnsums common sum files"
105	fi
106	exit $exit_status
107elif [ -d "$1" -o -d "$2" ] ; then
108	usage "Must specify either two directories or two files"
109fi
110
111sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
112sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
113
114before=$tmp1
115now=$tmp2
116
117
118if sort -k 2 </dev/null >/dev/null 2>&1; then
119  skip1='-k 2'
120else
121  skip1='+1'
122fi
123
124sort -t ':' $skip1 "$now" > "$now_s"
125sort -t ':' $skip1 "$before" > "$before_s"
126
127grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
128grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
129
130grep -s . $tmp2 >/dev/null
131if [ $? = 0 ]; then
132	num=`cat $tmp2 | wc -l`
133	echo "Tests that now fail, but worked before ($num tests):"
134	echo
135	cat $tmp2
136	echo
137	exit_status=1
138fi
139
140grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
141grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
142
143grep -s . $tmp2 >/dev/null
144if [ $? = 0 ]; then
145	num=`cat $tmp2 | wc -l`
146	echo "Tests that now work, but didn't before ($num tests):"
147	echo
148	cat $tmp2
149	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
150	echo
151fi
152
153grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
154grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
155
156grep -s . $tmp2 >/dev/null
157if [ $? = 0 ]; then
158	num=`cat $tmp2 | wc -l`
159	echo "New tests that FAIL ($num tests):"
160	echo
161	cat $tmp2
162	echo
163	exit_status=1
164fi
165
166grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
167grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
168
169grep -s . $tmp2 >/dev/null
170if [ $? = 0 ]; then
171	num=`cat $tmp2 | wc -l`
172	echo "New tests that PASS ($num tests):"
173	echo
174	cat $tmp2
175	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
176	echo
177fi
178
179grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
180grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
181
182grep -s . $tmp2 >/dev/null
183if [ $? = 0 ]; then
184	num=`cat $tmp2 | wc -l`
185	echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
186	echo
187	cat $tmp2
188	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
189	echo
190fi
191
192grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
193grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
194
195grep -s . $tmp2 >/dev/null
196if [ $? = 0 ]; then
197	num=`cat $tmp2 | wc -l`
198	echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
199	echo
200	cat $tmp2
201	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
202	echo
203fi
204
205exit $exit_status
206