1#!/usr/local/bin/bash
2
3# This runs a test while providing diagnostic output on error.
4# $1  The Frobby action to run (e.g. alexdual)
5# $2  The name of one or more space-seperated input files
6# $3  The reference output file
7# $4+ Parameters to this script and then parameters to frobby
8#
9# Parameter 2 can contain more than one file if this script has been called
10# as e.g. "../thisscript action "file1 file2" reference
11#
12# If the output produced by Frobby does not match that from the
13# reference output according to diff, then this is considered an
14# error, diagnostics will be printed and the exit status will be 1.
15# Otherwise a dot '.' is printed and the exit status is 0.
16#
17# If $4 is _generate, then the output from Frobby is piped into the
18# reference output file, overwriting any previous contents. It is
19# recommended to manually inspect the output to check if it is correct
20# and to keep a copy of the previous contents of the file.
21#
22# If $4 is _summary, then nothing will be printed on success, and a
23# short summary line is printed on failure. In this case the exit
24# status is zero even on failure. The intent of this is to get a
25# summary of all failing tests, instead of getting detailed
26# information on the first failure.
27#
28# If $4 is _valgrind, then frobby will be run under valgrind. If
29# valgrind reports no errors and no memory leaks, then a 'v' is
30# printed to indicate this, and the exist status is 0. If valgrind
31# does report an issue, then the valgrind output is displayed and the
32# exit status is 1. In either case the actual output of Frobby is
33# ignored and is thus not compared to the correct reference output.
34#
35# If $4 is _expectExitCode, then the expected error code is $5 instead
36# of the usual expectation of 0.
37#
38# If $4 is _matchError, then the output written to standard error is
39# matched against the reference output, instead of that written to
40# standard out. Lines containing the word DEBUG is removed from this
41# output.
42#
43# If $4 is _debugAlloc, then Frobby will be run with the _debugAlloc
44# option, which will make it run through every scenario of runnning
45# out of memory and recovering, which can take a long time. This only
46# works for the debug versions of Frobby.
47#
48# These options can all be combined by adding them one after each
49# other. In that case replace $4 in the descriptions above by the
50# appropriate parameter. The script stops looking for further
51# parameters as soon as it encounters a parameter it does not
52# understand, so script parameters have to be before parameters to
53# Frobby.
54#
55# This script is designed to be run from a sub-directory of where it
56# resides.
57
58origParams="$*" # used for debug output below
59origFrobby="../../bin/frobby"
60origFrobbyOut="./frobbyTestScriptTemporary_standardOutput"
61origFrobbyErr="./frobbyTestScriptTemporary_standardError"
62frobbyChangedErr="./frobbyTestScriptTemporary_standardErrorChanged"
63frobbyChangedInput="./frobbyTestScriptTemporary_standardInput"
64
65frobby="$origFrobby"
66frobbyOut="$origFrobbyOut"
67frobbyErr="$origFrobbyErr"
68action="$1"
69inputFile="$2"
70referenceOutputFile="$3"
71
72# Note that we *must* use
73#   echo $inputFile|...
74# instead of the simpler
75#   ... < $inputFile
76# because the latter does not work if $inputFile has more than one
77# file name in it.
78
79shift
80shift
81shift
82
83# Set initial values
84generate=0;
85expectedExitCode=0;
86matchErr=0;
87useValgrind=0;
88useDebugAlloc=0
89summary=0;
90
91changed=1;
92while [ $changed = 1 ];
93do
94  changed=0;
95
96  if [ "$1" = "_generate" ];
97  then
98    changed=1;
99    if [ "$inputFile" == "$referenceOutputFile" ]; then exit 0; fi
100    generate=1;
101    shift;
102  fi
103
104  if [ "$1" = "_summary" ];
105  then
106    changed=1;
107    summary=1;
108    shift;
109  fi
110
111  if [ "$1" = "_valgrind" ];
112  then
113    changed=1;
114    useValgrind=1;
115    shift;
116  fi
117
118  if [ "$1" = "_debugAlloc" ];
119  then
120    changed=1;
121    useDebugAlloc=1;
122    shift;
123  fi
124
125  if [ "$1" = "_expectExitCode" ];
126  then
127    changed=1;
128    expectedExitCode="$2";
129    shift;
130    shift;
131  fi
132
133  if [ "$1" = "_matchError" ];
134  then
135    changed=1;
136    matchErr=1;
137    shift;
138  fi
139done
140
141if [ $useDebugAlloc = 1 ];
142then
143  # note that we have to use the _input option, since Frobby starts
144  # the computation over after each simulated recovery, so it needs to
145  # be able to rewind the input too, which is not possible using
146  # pipes.  Also, since the _input notation only supports a single
147  # file, we have to collect all the input into a single file, in case
148  # $inputFile contains more than a single filename.
149  cat $inputFile > $frobbyChangedInput
150  origFrobby="$frobby _debugAlloc _input $frobbyChangedInput";
151  frobby="$origFrobby"
152  # OK, I change origFrobby, so it is not quite original. So shoot me.
153fi
154
155if [ $useValgrind = 1 ];
156then
157  frobby="valgrind $frobby";
158fi
159
160params="$*"
161
162# Set printDebugOutput to 1 outside the script to see this output.
163if [ "$printDebugOutput" == 1 ];
164then
165  echo "------ debug output ------"
166  echo "frobby is \"$frobby\""
167  echo "origParams is \"$origParams\""
168  echo "action is \"$action\""
169  echo "inputFile is \"$inputFile\""
170  echo "params is \"$params\""
171  echo "referenceOutputFile is \"$referenceOutputFile\""
172  echo "cmd is \"$cmd\""
173  echo "generate is $generate"
174  echo "summary is $summary"
175  echo "useValgrind is $useValgrind"
176  echo "useDebugAlloc is $useDebugAlloc"
177  echo "expectedExitCode is $expectedExitCode"
178  echo "matchErr is $matchErr"
179fi
180
181# This is where we actually run Frobby
182cat $inputFile|$frobby $action $params > $frobbyOut 2> $frobbyErr
183frobbyExitCode="$?"
184
185if [ $matchErr = 1 ];
186then
187  # Use a new file to remove lines containing the words DEBUG
188  frobbyOut="$frobbyChangedErr";
189  sed /DEBUG/d < $frobbyErr|sed /^==/d > $frobbyOut
190fi
191
192# Check if Frobby's output matches the reference output file.
193diff $frobbyOut $referenceOutputFile > /dev/null 2> /dev/null
194referenceMatch="$?"
195
196if [ $generate = 1 ];
197then
198  if [ $frobbyExitCode != $expectedExitCode ];
199  then
200    echo "Halting generation of output due to getting exit code $frobbyExitCode."
201    echo "The expected exit code was $expectedExitCode. This is for "
202	echo "  cat $inputFile|$frobby $action $params"
203    echo "Giving error output of "
204    cat $origFrobbyErr
205    exit 1
206  fi
207
208  cp -f $frobbyOut $referenceOutputFile;
209  if [ $referenceMatch = 0 ];
210  then
211    echo -n "g";
212  else
213    echo;
214    echo -n "Replaced output file $referenceOutputFile with different version.";
215  fi
216  exit 0
217fi
218
219if [ $useValgrind = 1 ];
220then
221  errs=`grep -e "ERROR SUMMARY: [^0].* error" \
222             -e "--leak-check=full" $frobbyErr`;
223  if [ "$errs" == "" ];
224  then
225    echo -n v
226  else
227    echo
228    echo
229    echo "valgrind reports issue on running:"
230    echo "  cat $inputFile|valgrind $frobby $action $params"
231    echo "Re-running to show details:"
232    echo
233
234    cat $inputFile|valgrind --leak-check=full --show-reachable=no $origFrobby $action $params > /dev/null;
235    echo "This is a valgrind report on running:"
236    echo "  cat $inputFile|valgrind $frobby $action $params"
237    exit 1;
238  fi
239fi
240
241# The option _debugAlloc makes Frobby rerun many times, and to avoid
242# making the same output many times, it closes standard out after one
243# run.  It does not close standard error, so matching on error and
244# _debugAlloc cannot be combined in a sensible way, so we stop that
245# from happening.
246if [[ $matchErr = 1 && $useDebugAlloc = 1 && $frobbyExitCode = $expectedExitCode ]]; then
247  echo -n "d";
248  exit 0;
249fi
250
251if [[ $referenceMatch = 0 && $frobbyExitCode = $expectedExitCode ]];
252then
253  if [ $summary = 0 ];
254  then
255    if [ $useDebugAlloc = 1 ];
256    then
257      echo -n ".d"
258    else
259      echo -n "."
260    fi
261  fi
262  exit 0;
263fi
264
265if [ $summary = 1 ];
266then
267  echo "Error on $action: $inputFile -> $referenceOutputFile";
268  exit 0;
269fi
270
271echo
272echo
273echo "***** Encountered failure on output $referenceOutputFile. Input head was:"
274cat $inputFile|head
275
276echo "***** Reference output head:"
277cat $referenceOutputFile|head
278
279echo "***** Standard output head from Frobby:"
280cat $origFrobbyOut|head
281
282echo "***** Error output head from Frobby:"
283cat $origFrobbyErr|head
284
285echo "***** Head of diff of output versus reference is"
286diff $referenceOutputFile $frobbyOut|head
287
288echo "***** Exit code was $frobbyExitCode, and expecting $expectedExitCode"
289if [ $matchErr = 1 ];
290then
291   echo "***** Matching on error output, not standard output.";
292fi
293
294echo
295echo "***** A test $action: $inputFile -> $referenceOutputFile failed."
296echo "  cat $inputFile|$frobby $action $params"
297
298exit 1
299