1#!/usr/local/bin/bash 2 3# This runs a test while providing diagnostic output on error. 4# $1 The Frobby action to run (e.g. alexdual) 5# $2 The name of one or more space-seperated input files 6# $3 The reference output file 7# $4+ Parameters to this script and then parameters to frobby 8# 9# Parameter 2 can contain more than one file if this script has been called 10# as e.g. "../thisscript action "file1 file2" reference 11# 12# If the output produced by Frobby does not match that from the 13# reference output according to diff, then this is considered an 14# error, diagnostics will be printed and the exit status will be 1. 15# Otherwise a dot '.' is printed and the exit status is 0. 16# 17# If $4 is _generate, then the output from Frobby is piped into the 18# reference output file, overwriting any previous contents. It is 19# recommended to manually inspect the output to check if it is correct 20# and to keep a copy of the previous contents of the file. 21# 22# If $4 is _summary, then nothing will be printed on success, and a 23# short summary line is printed on failure. In this case the exit 24# status is zero even on failure. The intent of this is to get a 25# summary of all failing tests, instead of getting detailed 26# information on the first failure. 27# 28# If $4 is _valgrind, then frobby will be run under valgrind. If 29# valgrind reports no errors and no memory leaks, then a 'v' is 30# printed to indicate this, and the exist status is 0. If valgrind 31# does report an issue, then the valgrind output is displayed and the 32# exit status is 1. In either case the actual output of Frobby is 33# ignored and is thus not compared to the correct reference output. 34# 35# If $4 is _expectExitCode, then the expected error code is $5 instead 36# of the usual expectation of 0. 37# 38# If $4 is _matchError, then the output written to standard error is 39# matched against the reference output, instead of that written to 40# standard out. Lines containing the word DEBUG is removed from this 41# output. 42# 43# If $4 is _debugAlloc, then Frobby will be run with the _debugAlloc 44# option, which will make it run through every scenario of runnning 45# out of memory and recovering, which can take a long time. This only 46# works for the debug versions of Frobby. 47# 48# These options can all be combined by adding them one after each 49# other. In that case replace $4 in the descriptions above by the 50# appropriate parameter. The script stops looking for further 51# parameters as soon as it encounters a parameter it does not 52# understand, so script parameters have to be before parameters to 53# Frobby. 54# 55# This script is designed to be run from a sub-directory of where it 56# resides. 57 58origParams="$*" # used for debug output below 59origFrobby="../../bin/frobby" 60origFrobbyOut="./frobbyTestScriptTemporary_standardOutput" 61origFrobbyErr="./frobbyTestScriptTemporary_standardError" 62frobbyChangedErr="./frobbyTestScriptTemporary_standardErrorChanged" 63frobbyChangedInput="./frobbyTestScriptTemporary_standardInput" 64 65frobby="$origFrobby" 66frobbyOut="$origFrobbyOut" 67frobbyErr="$origFrobbyErr" 68action="$1" 69inputFile="$2" 70referenceOutputFile="$3" 71 72# Note that we *must* use 73# echo $inputFile|... 74# instead of the simpler 75# ... < $inputFile 76# because the latter does not work if $inputFile has more than one 77# file name in it. 78 79shift 80shift 81shift 82 83# Set initial values 84generate=0; 85expectedExitCode=0; 86matchErr=0; 87useValgrind=0; 88useDebugAlloc=0 89summary=0; 90 91changed=1; 92while [ $changed = 1 ]; 93do 94 changed=0; 95 96 if [ "$1" = "_generate" ]; 97 then 98 changed=1; 99 if [ "$inputFile" == "$referenceOutputFile" ]; then exit 0; fi 100 generate=1; 101 shift; 102 fi 103 104 if [ "$1" = "_summary" ]; 105 then 106 changed=1; 107 summary=1; 108 shift; 109 fi 110 111 if [ "$1" = "_valgrind" ]; 112 then 113 changed=1; 114 useValgrind=1; 115 shift; 116 fi 117 118 if [ "$1" = "_debugAlloc" ]; 119 then 120 changed=1; 121 useDebugAlloc=1; 122 shift; 123 fi 124 125 if [ "$1" = "_expectExitCode" ]; 126 then 127 changed=1; 128 expectedExitCode="$2"; 129 shift; 130 shift; 131 fi 132 133 if [ "$1" = "_matchError" ]; 134 then 135 changed=1; 136 matchErr=1; 137 shift; 138 fi 139done 140 141if [ $useDebugAlloc = 1 ]; 142then 143 # note that we have to use the _input option, since Frobby starts 144 # the computation over after each simulated recovery, so it needs to 145 # be able to rewind the input too, which is not possible using 146 # pipes. Also, since the _input notation only supports a single 147 # file, we have to collect all the input into a single file, in case 148 # $inputFile contains more than a single filename. 149 cat $inputFile > $frobbyChangedInput 150 origFrobby="$frobby _debugAlloc _input $frobbyChangedInput"; 151 frobby="$origFrobby" 152 # OK, I change origFrobby, so it is not quite original. So shoot me. 153fi 154 155if [ $useValgrind = 1 ]; 156then 157 frobby="valgrind $frobby"; 158fi 159 160params="$*" 161 162# Set printDebugOutput to 1 outside the script to see this output. 163if [ "$printDebugOutput" == 1 ]; 164then 165 echo "------ debug output ------" 166 echo "frobby is \"$frobby\"" 167 echo "origParams is \"$origParams\"" 168 echo "action is \"$action\"" 169 echo "inputFile is \"$inputFile\"" 170 echo "params is \"$params\"" 171 echo "referenceOutputFile is \"$referenceOutputFile\"" 172 echo "cmd is \"$cmd\"" 173 echo "generate is $generate" 174 echo "summary is $summary" 175 echo "useValgrind is $useValgrind" 176 echo "useDebugAlloc is $useDebugAlloc" 177 echo "expectedExitCode is $expectedExitCode" 178 echo "matchErr is $matchErr" 179fi 180 181# This is where we actually run Frobby 182cat $inputFile|$frobby $action $params > $frobbyOut 2> $frobbyErr 183frobbyExitCode="$?" 184 185if [ $matchErr = 1 ]; 186then 187 # Use a new file to remove lines containing the words DEBUG 188 frobbyOut="$frobbyChangedErr"; 189 sed /DEBUG/d < $frobbyErr|sed /^==/d > $frobbyOut 190fi 191 192# Check if Frobby's output matches the reference output file. 193diff $frobbyOut $referenceOutputFile > /dev/null 2> /dev/null 194referenceMatch="$?" 195 196if [ $generate = 1 ]; 197then 198 if [ $frobbyExitCode != $expectedExitCode ]; 199 then 200 echo "Halting generation of output due to getting exit code $frobbyExitCode." 201 echo "The expected exit code was $expectedExitCode. This is for " 202 echo " cat $inputFile|$frobby $action $params" 203 echo "Giving error output of " 204 cat $origFrobbyErr 205 exit 1 206 fi 207 208 cp -f $frobbyOut $referenceOutputFile; 209 if [ $referenceMatch = 0 ]; 210 then 211 echo -n "g"; 212 else 213 echo; 214 echo -n "Replaced output file $referenceOutputFile with different version."; 215 fi 216 exit 0 217fi 218 219if [ $useValgrind = 1 ]; 220then 221 errs=`grep -e "ERROR SUMMARY: [^0].* error" \ 222 -e "--leak-check=full" $frobbyErr`; 223 if [ "$errs" == "" ]; 224 then 225 echo -n v 226 else 227 echo 228 echo 229 echo "valgrind reports issue on running:" 230 echo " cat $inputFile|valgrind $frobby $action $params" 231 echo "Re-running to show details:" 232 echo 233 234 cat $inputFile|valgrind --leak-check=full --show-reachable=no $origFrobby $action $params > /dev/null; 235 echo "This is a valgrind report on running:" 236 echo " cat $inputFile|valgrind $frobby $action $params" 237 exit 1; 238 fi 239fi 240 241# The option _debugAlloc makes Frobby rerun many times, and to avoid 242# making the same output many times, it closes standard out after one 243# run. It does not close standard error, so matching on error and 244# _debugAlloc cannot be combined in a sensible way, so we stop that 245# from happening. 246if [[ $matchErr = 1 && $useDebugAlloc = 1 && $frobbyExitCode = $expectedExitCode ]]; then 247 echo -n "d"; 248 exit 0; 249fi 250 251if [[ $referenceMatch = 0 && $frobbyExitCode = $expectedExitCode ]]; 252then 253 if [ $summary = 0 ]; 254 then 255 if [ $useDebugAlloc = 1 ]; 256 then 257 echo -n ".d" 258 else 259 echo -n "." 260 fi 261 fi 262 exit 0; 263fi 264 265if [ $summary = 1 ]; 266then 267 echo "Error on $action: $inputFile -> $referenceOutputFile"; 268 exit 0; 269fi 270 271echo 272echo 273echo "***** Encountered failure on output $referenceOutputFile. Input head was:" 274cat $inputFile|head 275 276echo "***** Reference output head:" 277cat $referenceOutputFile|head 278 279echo "***** Standard output head from Frobby:" 280cat $origFrobbyOut|head 281 282echo "***** Error output head from Frobby:" 283cat $origFrobbyErr|head 284 285echo "***** Head of diff of output versus reference is" 286diff $referenceOutputFile $frobbyOut|head 287 288echo "***** Exit code was $frobbyExitCode, and expecting $expectedExitCode" 289if [ $matchErr = 1 ]; 290then 291 echo "***** Matching on error output, not standard output."; 292fi 293 294echo 295echo "***** A test $action: $inputFile -> $referenceOutputFile failed." 296echo " cat $inputFile|$frobby $action $params" 297 298exit 1 299