1#!/bin/sh
2
3# Tests a set of patches from a directory.
4# Copyright (C) 2007, 2008, 2011  Free Software Foundation, Inc.
5# Contributed by Sebastian Pop <sebastian.pop@amd.com>
6
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 3 of the License, or
10# (at your option) any later version.
11
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16
17# You should have received a copy of the GNU General Public License
18# along with this program; if not, write to the Free Software
19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
20
21cat <<EOF
22
23WARNING: This script should only be fed with patches from known
24         authorized and trusted sources.  Don't even think about
25         hooking it up to a raw feed from the gcc-patches list or
26         you'll regret it.
27
28EOF
29
30args=$@
31
32svnpath=svn://gcc.gnu.org/svn/gcc
33dashj=
34default_standby=1
35standby=$default_standby
36default_watermark=0.60
37watermark=$default_watermark
38savecompilers=false
39nopristinecache=false
40nogpg=false
41stop=false
42
43usage() {
44    cat <<EOF
45patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
46                [-svnpath URL] [-stop] [-nopristinecache]
47                <source_dir> [patches_dir [state_dir [build_dir]]]
48
49    J is the flag passed to make.  Default is empty string.
50
51    STANDBY is the number of minutes between checks for new patches in
52    PATCHES_DIR.  Default is ${default_standby} minutes.
53
54    WATERMARK is the 5 minute average system charge under which a new
55    compile can start.  Default is ${default_watermark}.
56
57    SAVECOMPILERS copies the compilers in the same directory as the
58    test results for the non patched version.  Default is not copy.
59
60    NOPRISTINECACHE prevents use of cached test results from any earlier
61    test runs on the pristine version of the branch and revision under
62    test (the default behaviour).  This should be used when testing the
63    same revision and patch with multiple sets of configure options, as
64    these may affect the set of baseline failures.
65
66    NOGPG can be used to avoid checking the GPG signature of patches.
67
68    URL is the location of the GCC SVN repository.  The default is
69    ${svnpath}.
70
71    STOP exits when PATCHES_DIR is empty.
72
73    SOURCE_DIR is the directory containing GCC's toplevel configure.
74
75    PATCHES_DIR is the directory containing the patches to be tested.
76    Default is SOURCE_DIR/patches.
77
78    STATE_DIR is where the tester maintains its internal state.
79    Default is SOURCE_DIR/state.
80
81    BUILD_DIR is the build tree, a temporary directory that this
82    script will delete and recreate.  Default is SOURCE_DIR/obj.
83
84EOF
85    exit 1
86}
87
88makedir () {
89    DIRNAME=$1
90    mkdir -p $DIRNAME
91    if [ $? -ne 0 ]; then
92	echo "ERROR: could not make directory $DIRNAME"
93	exit 1
94    fi
95}
96
97while [ $# -ne 0 ]; do
98    case $1 in
99	-j*)
100	    dashj=$1; shift
101	    ;;
102	-standby)
103	    [[ $# > 2 ]] || usage
104	    standby=$2; shift; shift
105	    ;;
106	-watermark)
107	    [[ $# > 2 ]] || usage
108	    watermark=$2; shift; shift
109	    ;;
110	-savecompilers)
111	    savecompilers=true; shift
112	    ;;
113	-nopristinecache)
114	    nopristinecache=true; shift
115	    ;;
116	-nogpg)
117	    nogpg=true; shift
118	    ;;
119	-stop)
120	    stop=true; shift
121	    ;;
122	-svnpath)
123	    svnpath=$2; shift; shift
124	    ;;
125	-*)
126	    echo "Invalid option: $1"
127	    usage
128	    ;;
129	*)
130	    break
131	    ;;
132    esac
133done
134
135test $# -eq 0 && usage
136
137SOURCE=$1
138PATCHES=
139STATE=
140BUILD=
141
142if [[ $# < 2 ]]; then
143    PATCHES=$SOURCE/patches
144else
145    PATCHES=$2
146fi
147if [[ $# < 3 ]]; then
148    STATE=$SOURCE/state
149else
150    STATE=$3
151fi
152if [[ $# < 4 ]]; then
153    BUILD=$SOURCE/obj
154else
155    BUILD=$4
156fi
157
158[ -d $PATCHES ] || makedir $PATCHES
159[ -d $STATE ] || makedir $STATE
160[ -d $STATE/patched ] || makedir $STATE/patched
161[ -d $SOURCE ] || makedir $SOURCE
162[ -f $SOURCE/config.guess ] || {
163    cd $SOURCE
164    svn -q co $svnpath/trunk .
165    if [ $? -ne 0 ]; then
166	echo "ERROR: initial svn checkout failed"
167	exit 1
168    fi
169}
170
171# This can contain required local settings:
172#  default_config  configure options, always passed
173#  default_make    make bootstrap options, always passed
174#  default_check   make check options, always passed
175[ -f $STATE/defaults ] && . $STATE/defaults
176
177VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
178
179exec >> $STATE/tester.log 2>&1 || exit 1
180set -x
181
182TESTING=$STATE/testing
183REPORT=$TESTING/report
184PRISTINE=$TESTING/pristine
185PATCHED=$TESTING/patched
186PATCH=
187TARGET=`$SOURCE/config.guess || exit 1`
188TESTLOGS="gcc/testsuite/gcc/gcc.sum
189gcc/testsuite/gfortran/gfortran.sum
190gcc/testsuite/g++/g++.sum
191gcc/testsuite/objc/objc.sum
192$TARGET/libstdc++-v3/testsuite/libstdc++.sum
193$TARGET/libffi/testsuite/libffi.sum
194$TARGET/libgomp/testsuite/libgomp.sum
195$TARGET/libmudflap/testsuite/libmudflap.sum"
196COMPILERS="gcc/cc1
197gcc/cc1obj
198gcc/cc1plus
199gcc/f951
200gcc/jc1
201gcc/gnat1
202gcc/tree1"
203
204now () {
205    echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
206}
207
208report () {
209    echo "$@" >> $REPORT
210}
211
212freport () {
213    if [ -s $1 ]; then
214	report "(cat $1"
215	cat $1 >> $REPORT
216	report "tac)"
217    fi
218}
219
220cleanup () {
221    cd $SOURCE
222    svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
223}
224
225selfexec () {
226    exec ${CONFIG_SHELL-/bin/sh} $0 $args
227}
228
229update () {
230    svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
231    if [ x$svn_branch = x ]; then
232	svn_branch=trunk
233    fi
234
235    svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
236    if [ x$svn_revision = x ]; then
237	svn_revision=HEAD
238    fi
239
240    cleanup
241    cd $SOURCE
242    case $svn_branch in
243	trunk)
244	    if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
245		report "failed to update svn sources with"
246		report "svn switch -r $svn_revision $svnpath/trunk"
247		freport $TESTING/svn
248		return 1
249	    fi
250	    ;;
251
252	${svnpath}*)
253	    if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
254		report "failed to update svn sources with"
255		report "svn switch -r $svn_revision $svn_branch"
256		freport $TESTING/svn
257		return 1
258	    fi
259	    ;;
260
261	*)
262	    if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
263		report "failed to update svn sources with"
264		report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
265		freport $TESTING/svn
266		return 1
267	    fi
268	    ;;
269    esac
270    contrib/gcc_update --touch
271
272    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
273    if [[ $VERSION < $current_version ]]; then
274	if [ -f $SOURCE/contrib/patch_tester.sh ]; then
275	    selfexec
276	fi
277    fi
278
279    return 0
280}
281
282apply_patch () {
283    if [ $nogpg = false ]; then
284	if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
285	    report "your patch failed to verify:"
286	    freport $TESTING/gpgverify
287	    return 1
288	fi
289    fi
290
291    cd $SOURCE
292    if ! patch -p0 < $PATCH &> $TESTING/patching ; then
293	report "your patch failed to apply:"
294	report "(check that the patch was created at the top level)"
295	freport $TESTING/patching
296	return 1
297    fi
298
299    # Just assume indexes for now -- not really great, but svn always
300    # makes them.
301    grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
302	# If the patch resulted in an empty file, delete it.
303	# This is how svn reports deletions.
304	if [ ! -s $file ]; then
305	    rm -f $file
306	    report "Deleting empty file $file"
307	fi
308    done
309}
310
311save_compilers () {
312    for COMPILER in $COMPILERS ; do
313	if [ -f $BUILD/$COMPILER ]; then
314	    cp $BUILD/$COMPILER $PRISTINE
315	fi
316    done
317}
318
319bootntest () {
320    rm -rf $BUILD
321    mkdir $BUILD
322    cd $BUILD
323
324    CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
325    CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
326    if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
327	report "configure with `basename $1` version failed with:"
328	freport $1/configure
329	return 1
330    fi
331
332    MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
333    MAKE_ARGS="$default_make $MAKE_ARGS"
334    if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
335	report "bootstrap with `basename $1` version failed with last lines:"
336	tail -30 $1/bootstrap > $1/last_bootstrap
337	freport $1/last_bootstrap
338	report "grep --context=20 Error bootstrap:"
339	grep --context=20 Error $1/bootstrap > $1/bootstrap_error
340	freport $1/bootstrap_error
341	return 1
342    fi
343
344    CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
345    CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
346    eval make $dashj $CHECK_OPTIONS -k check &> $1/check
347
348    SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
349    if [ x$SUITESRUN = x ]; then
350	report "check with `basename $1` version failed, no testsuites were run"
351	return 1
352    fi
353
354    for LOG in $TESTLOGS ; do
355	if [ -f $BUILD/$LOG ]; then
356	    mv $BUILD/$LOG $1
357	    mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
358	fi
359    done
360
361    return 0
362}
363
364bootntest_patched () {
365    cleanup
366    mkdir -p $PATCHED
367    apply_patch && bootntest $PATCHED
368    return $?
369}
370
371# Build the pristine tree with exactly the same options as the patch under test.
372bootntest_pristine () {
373    cleanup
374    current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
375    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
376    PRISTINE=$STATE/$current_branch/$current_version
377
378    if [ $nopristinecache = true ]; then
379      rm -rf $PRISTINE
380    fi
381    if [ -d $PRISTINE ]; then
382	ln -s $PRISTINE $TESTING/pristine
383	return 0
384    else
385	mkdir -p $PRISTINE
386	ln -s $PRISTINE $TESTING/pristine
387	bootntest $PRISTINE
388	RETVAL=$?
389	if [ $RETVAL = 0 -a $savecompilers = true ]; then
390	    save_compilers
391	fi
392	return $RETVAL
393    fi
394}
395
396regtest () {
397    touch $1/report
398    touch $1/passes
399    touch $1/failed
400    touch $1/regress
401
402    for LOG in $TESTLOGS ; do
403	NLOG=`basename $LOG`
404	if [ -f $1/$NLOG ]; then
405	    awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
406	fi
407    done | sort | uniq > $1/failed
408
409    comm -12 $1/failed $1/passes >> $1/regress
410    NUMREGRESS=`wc -l < $1/regress | tr -d ' '`
411
412    if [ $NUMREGRESS -eq 0 ] ; then
413	for LOG in $TESTLOGS ; do
414	    NLOG=`basename $LOG`
415	    if [ -f $1/$NLOG ] ; then
416		awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
417	    fi
418	done | sort | uniq | comm -23 - $1/failed > $1/passes
419	echo "there are no regressions with your patch." >> $1/report
420    else
421	echo "with your patch there are $NUMREGRESS regressions." >> $1/report
422	echo "list of regressions with your patch:" >> $1/report
423	cat $1/regress >> $1/report
424    fi
425}
426
427contrib_compare_tests () {
428    report "comparing logs with contrib/compare_tests:"
429    for LOG in $TESTLOGS ; do
430 	NLOG=`basename $LOG`
431 	if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
432 	    $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
433 	    freport $TESTING/compare_$NLOG
434 	fi
435    done
436}
437
438compare_passes () {
439    regtest $PRISTINE
440    cp $PRISTINE/passes $PATCHED
441    regtest $PATCHED
442    freport $PATCHED/report
443    report "FAILs with patched version:"
444    freport $PATCHED/failed
445    report "FAILs with pristine version:"
446    freport $PRISTINE/failed
447
448    # contrib_compare_tests
449}
450
451write_report () {
452    backup_patched=$STATE/patched/`now`
453    report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."
454
455    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
456    if [ x$EMAIL != x ]; then
457	mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
458    fi
459
460    mv $TESTING $backup_patched
461}
462
463announce () {
464    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
465    if [ x$EMAIL != x ]; then
466
467	START_REPORT=$TESTING/start_report
468	echo "Hi, " >> $START_REPORT
469	echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
470	echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
471	echo "Bye, your automatic tester." >> $START_REPORT
472	mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
473    fi
474}
475
476# After selfexec, $TESTING is already set up.
477if [ -d $TESTING ]; then
478    # The only file in $TESTING is the patch.
479    PATCH=`ls -rt -1 $TESTING | head -1`
480    PATCH=$TESTING/$PATCH
481    if [ -f $PATCH ]; then
482	bootntest_patched && bootntest_pristine && compare_passes
483	write_report
484    fi
485fi
486
487firstpatch=true
488while true; do
489    PATCH=`ls -rt -1 $PATCHES | head -1`
490    if [ x$PATCH = x ]; then
491	if [ $stop = true ]; then
492	    if [ $firstpatch = true ]; then
493		echo "No patches ready to test, quitting."
494		exit 1
495	    else
496		echo "No more patches to test."
497		exit 0
498	    fi
499	fi
500	sleep ${standby}m
501    else
502	firstpatch=false
503	sysload=`uptime | cut -d, -f 5`
504	if [[ $sysload > $watermark ]]; then
505	    # Wait a bit when system load is too high.
506	    sleep ${standby}m
507	else
508	    mkdir -p $TESTING
509	    mv $PATCHES/$PATCH $TESTING/
510	    PATCH=$TESTING/$PATCH
511
512	    announce
513	    update && bootntest_patched && bootntest_pristine && compare_passes
514	    write_report
515	fi
516    fi
517done
518