1#!/bin/sh
2# This script automatically test the given tool with the tool's test cases,
3# reporting anything of interest.
4
5# Written by Mike Stump <mrs@cygnus.com>
6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
7
8usage()
9{
10	if [ -n "$1" ] ; then
11		echo "$0: Error: $1" >&2
12		echo >&2
13	fi
14	cat >&2 <<EOUSAGE
15Usage: $0 [-strict] PREVIOUS CURRENT
16
17Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
18
19	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
20
21	Unless -strict is given, these discrepancies are not counted as errors:
22		missing/extra .sum files when comparing directories
23		tests that failed in PREVIOUS but pass in CURRENT
24		tests that were not in PREVIOUS but appear in CURRENT
25		tests in PREVIOUS that are missing in CURRENT
26
27	Exit with the following values:
28		0 if there is nothing of interest
29		1 if there are errors when comparing single test case files
30		N for the number of errors found when comparing directories
31EOUSAGE
32	exit 2
33}
34
35export LC_ALL=C
36
37tool=gxx
38
39TMPDIR=${TMPDIR:-/tmp}
40tmp1=$TMPDIR/$tool-testing.$$a
41tmp2=$TMPDIR/$tool-testing.$$b
42now_s=$TMPDIR/$tool-testing.$$d
43before_s=$TMPDIR/$tool-testing.$$e
44lst1=$TMPDIR/$tool-lst1.$$
45lst2=$TMPDIR/$tool-lst2.$$
46lst3=$TMPDIR/$tool-lst3.$$
47lst4=$TMPDIR/$tool-lst4.$$
48lst5=$TMPDIR/$tool-lst5.$$
49sum1=$TMPDIR/$tool-sum1.$$
50sum2=$TMPDIR/$tool-sum2.$$
51tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
52
53[ "$1" = "-strict" ] && strict=$1 && shift
54[ "$1" = "-?" ] && usage
55[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
56
57trap "rm -f $tmps" 0 1 2 3 5 9 13 15
58exit_status=0
59
60if [ -d "$1" -a -d "$2" ] ; then
61	find "$1/" -name '*.sum' >$lst1
62	find "$2/" -name '*.sum' >$lst2
63	echo "# Comparing directories"
64	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
65	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
66	echo
67	# remove leading directory components to compare
68	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
69	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
70	comm -23 $lst3 $lst4 >$lst5
71	if [ -s $lst5 ] ; then
72		echo "# Extra sum files in Dir1=$1"
73		sed -e "s|^|< $1/|" $lst5
74		echo
75		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
76	fi
77	comm -13 $lst3 $lst4 >$lst5
78	if [ -s $lst5 ] ; then
79		echo "# Extra sum files in Dir2=$2"
80		sed -e "s|^|> $2/|" $lst5
81		echo
82		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
83	fi
84	comm -12 $lst3 $lst4 | sort -u >$lst5
85	if [ ! -s $lst5 ] ; then
86		echo "# No common sum files"
87		exit_status=`expr $exit_status + 1`
88		exit $exit_status
89	fi
90	cmnsums=`cat $lst5 | wc -l`
91	echo "# Comparing $cmnsums common sum files"
92	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
93	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
94	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
95	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
96	ret=$?
97	if [ $ret -ne 0 ]; then
98		exit_status=`expr $exit_status + 1`
99		echo "## Differences found: $fname"
100	fi
101	if [ $exit_status -ne 0 ]; then
102		echo "# $exit_status differences in $cmnsums common sum files found"
103	else
104		echo "# No differences found in $cmnsums common sum files"
105	fi
106	exit $exit_status
107elif [ -d "$1" -o -d "$2" ] ; then
108	usage "Must specify either two directories or two files"
109fi
110
111sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
112sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
113
114before=$tmp1
115now=$tmp2
116
117
118if sort -k 2 </dev/null >/dev/null 2>&1; then
119  skip1='-k 2'
120else
121  skip1='+1'
122fi
123
124sort -t ':' $skip1 "$now" > "$now_s"
125sort -t ':' $skip1 "$before" > "$before_s"
126
127grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
128grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
129
130grep -s . $tmp2 >/dev/null
131if [ $? = 0 ]; then
132	echo "Tests that now fail, but worked before:"
133	echo
134	cat $tmp2
135	echo
136	exit_status=1
137fi
138
139grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
140grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
141
142grep -s . $tmp2 >/dev/null
143if [ $? = 0 ]; then
144	echo "Tests that now work, but didn't before:"
145	echo
146	cat $tmp2
147	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
148	echo
149fi
150
151grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
152grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
153
154grep -s . $tmp2 >/dev/null
155if [ $? = 0 ]; then
156	echo "New tests that FAIL:"
157	echo
158	cat $tmp2
159	echo
160	exit_status=1
161fi
162
163grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
164grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
165
166grep -s . $tmp2 >/dev/null
167if [ $? = 0 ]; then
168	echo "New tests that PASS:"
169	echo
170	cat $tmp2
171	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
172	echo
173fi
174
175grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
176grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
177
178grep -s . $tmp2 >/dev/null
179if [ $? = 0 ]; then
180	echo "Old tests that passed, that have disappeared: (Eeek!)"
181	echo
182	cat $tmp2
183	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
184	echo
185fi
186
187grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
188grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
189
190grep -s . $tmp2 >/dev/null
191if [ $? = 0 ]; then
192	echo "Old tests that failed, that have disappeared: (Eeek!)"
193	echo
194	cat $tmp2
195	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
196	echo
197fi
198
199exit $exit_status
200