1#!/bin/sh 2# This script automatically test the given tool with the tool's test cases, 3# reporting anything of interest. 4 5# Written by Mike Stump <mrs@cygnus.com> 6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com> 7 8usage() 9{ 10 if [ -n "$1" ] ; then 11 echo "$0: Error: $1" >&2 12 echo >&2 13 fi 14 cat >&2 <<EOUSAGE 15Usage: $0 [-strict] PREVIOUS CURRENT 16 17Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. 18 19 If PREVIOUS and CURRENT are directories, find and compare any *.sum files. 20 21 Unless -strict is given, these discrepancies are not counted as errors: 22 missing/extra .sum files when comparing directories 23 tests that failed in PREVIOUS but pass in CURRENT 24 tests that were not in PREVIOUS but appear in CURRENT 25 tests in PREVIOUS that are missing in CURRENT 26 27 Exit with the following values: 28 0 if there is nothing of interest 29 1 if there are errors when comparing single test case files 30 N for the number of errors found when comparing directories 31EOUSAGE 32 exit 2 33} 34 35export LC_ALL=C 36 37tool=gxx 38 39TMPDIR=${TMPDIR:-/tmp} 40tmp1=$TMPDIR/$tool-testing.$$a 41tmp2=$TMPDIR/$tool-testing.$$b 42now_s=$TMPDIR/$tool-testing.$$d 43before_s=$TMPDIR/$tool-testing.$$e 44lst1=$TMPDIR/$tool-lst1.$$ 45lst2=$TMPDIR/$tool-lst2.$$ 46lst3=$TMPDIR/$tool-lst3.$$ 47lst4=$TMPDIR/$tool-lst4.$$ 48lst5=$TMPDIR/$tool-lst5.$$ 49sum1=$TMPDIR/$tool-sum1.$$ 50sum2=$TMPDIR/$tool-sum2.$$ 51tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" 52 53[ "$1" = "-strict" ] && strict=$1 && shift 54[ "$1" = "-?" ] && usage 55[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" 56 57trap "rm -f $tmps" 0 1 2 3 5 9 13 15 58exit_status=0 59 60if [ -d "$1" -a -d "$2" ] ; then 61 find "$1/" -name '*.sum' >$lst1 62 find "$2/" -name '*.sum' >$lst2 63 echo "# Comparing directories" 64 echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" 65 echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" 66 echo 67 # remove leading directory components to compare 68 sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 69 sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 70 comm -23 $lst3 $lst4 >$lst5 71 if [ -s $lst5 ] ; then 72 echo "# Extra sum files in Dir1=$1" 73 sed -e "s|^|< $1/|" $lst5 74 echo 75 [ -n "$strict" ] && exit_status=`expr $exit_status + 1` 76 fi 77 comm -13 $lst3 $lst4 >$lst5 78 if [ -s $lst5 ] ; then 79 echo "# Extra sum files in Dir2=$2" 80 sed -e "s|^|> $2/|" $lst5 81 echo 82 [ -n "$strict" ] && exit_status=`expr $exit_status + 1` 83 fi 84 comm -12 $lst3 $lst4 | sort -u >$lst5 85 if [ ! -s $lst5 ] ; then 86 echo "# No common sum files" 87 exit_status=`expr $exit_status + 1` 88 exit $exit_status 89 fi 90 cmnsums=`cat $lst5 | wc -l` 91 echo "# Comparing $cmnsums common sum files" 92 ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 93 ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 94 echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" 95 ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 96 ret=$? 97 if [ $ret -ne 0 ]; then 98 exit_status=`expr $exit_status + 1` 99 echo "## Differences found: $fname" 100 fi 101 if [ $exit_status -ne 0 ]; then 102 echo "# $exit_status differences in $cmnsums common sum files found" 103 else 104 echo "# No differences found in $cmnsums common sum files" 105 fi 106 exit $exit_status 107elif [ -d "$1" -o -d "$2" ] ; then 108 usage "Must specify either two directories or two files" 109fi 110 111sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1 112sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2 113 114before=$tmp1 115now=$tmp2 116 117 118if sort -k 2 </dev/null >/dev/null 2>&1; then 119 skip1='-k 2' 120else 121 skip1='+1' 122fi 123 124sort -t ':' $skip1 "$now" > "$now_s" 125sort -t ':' $skip1 "$before" > "$before_s" 126 127grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 128grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 129 130grep -s . $tmp2 >/dev/null 131if [ $? = 0 ]; then 132 num=`cat $tmp2 | wc -l` 133 echo "Tests that now fail, but worked before ($num tests):" 134 echo 135 cat $tmp2 136 echo 137 exit_status=1 138fi 139 140grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 141grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 142 143grep -s . $tmp2 >/dev/null 144if [ $? = 0 ]; then 145 num=`cat $tmp2 | wc -l` 146 echo "Tests that now work, but didn't before ($num tests):" 147 echo 148 cat $tmp2 149 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 150 echo 151fi 152 153grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 154grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 155 156grep -s . $tmp2 >/dev/null 157if [ $? = 0 ]; then 158 num=`cat $tmp2 | wc -l` 159 echo "New tests that FAIL ($num tests):" 160 echo 161 cat $tmp2 162 echo 163 exit_status=1 164fi 165 166grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 167grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 168 169grep -s . $tmp2 >/dev/null 170if [ $? = 0 ]; then 171 num=`cat $tmp2 | wc -l` 172 echo "New tests that PASS ($num tests):" 173 echo 174 cat $tmp2 175 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 176 echo 177fi 178 179grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 180grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 181 182grep -s . $tmp2 >/dev/null 183if [ $? = 0 ]; then 184 num=`cat $tmp2 | wc -l` 185 echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)" 186 echo 187 cat $tmp2 188 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 189 echo 190fi 191 192grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 193grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 194 195grep -s . $tmp2 >/dev/null 196if [ $? = 0 ]; then 197 num=`cat $tmp2 | wc -l` 198 echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)" 199 echo 200 cat $tmp2 201 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 202 echo 203fi 204 205exit $exit_status 206