1#!/bin/sh 2 3# Tests a set of patches from a directory. 4# Copyright (C) 2007, 2008, 2011 Free Software Foundation, Inc. 5# Contributed by Sebastian Pop <sebastian.pop@amd.com> 6 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 3 of the License, or 10# (at your option) any later version. 11 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16 17# You should have received a copy of the GNU General Public License 18# along with this program; if not, write to the Free Software 19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 21cat <<EOF 22 23WARNING: This script should only be fed with patches from known 24 authorized and trusted sources. Don't even think about 25 hooking it up to a raw feed from the gcc-patches list or 26 you'll regret it. 27 28EOF 29 30args=$@ 31 32svnpath=svn://gcc.gnu.org/svn/gcc 33dashj= 34default_standby=1 35standby=$default_standby 36default_watermark=0.60 37watermark=$default_watermark 38savecompilers=false 39nopristinecache=false 40nogpg=false 41stop=false 42 43usage() { 44 cat <<EOF 45patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg] 46 [-svnpath URL] [-stop] [-nopristinecache] 47 <source_dir> [patches_dir [state_dir [build_dir]]] 48 49 J is the flag passed to make. Default is empty string. 50 51 STANDBY is the number of minutes between checks for new patches in 52 PATCHES_DIR. Default is ${default_standby} minutes. 53 54 WATERMARK is the 5 minute average system charge under which a new 55 compile can start. Default is ${default_watermark}. 56 57 SAVECOMPILERS copies the compilers in the same directory as the 58 test results for the non patched version. Default is not copy. 59 60 NOPRISTINECACHE prevents use of cached test results from any earlier 61 test runs on the pristine version of the branch and revision under 62 test (the default behaviour). This should be used when testing the 63 same revision and patch with multiple sets of configure options, as 64 these may affect the set of baseline failures. 65 66 NOGPG can be used to avoid checking the GPG signature of patches. 67 68 URL is the location of the GCC SVN repository. The default is 69 ${svnpath}. 70 71 STOP exits when PATCHES_DIR is empty. 72 73 SOURCE_DIR is the directory containing GCC's toplevel configure. 74 75 PATCHES_DIR is the directory containing the patches to be tested. 76 Default is SOURCE_DIR/patches. 77 78 STATE_DIR is where the tester maintains its internal state. 79 Default is SOURCE_DIR/state. 80 81 BUILD_DIR is the build tree, a temporary directory that this 82 script will delete and recreate. Default is SOURCE_DIR/obj. 83 84EOF 85 exit 1 86} 87 88makedir () { 89 DIRNAME=$1 90 mkdir -p $DIRNAME 91 if [ $? -ne 0 ]; then 92 echo "ERROR: could not make directory $DIRNAME" 93 exit 1 94 fi 95} 96 97while [ $# -ne 0 ]; do 98 case $1 in 99 -j*) 100 dashj=$1; shift 101 ;; 102 -standby) 103 [[ $# > 2 ]] || usage 104 standby=$2; shift; shift 105 ;; 106 -watermark) 107 [[ $# > 2 ]] || usage 108 watermark=$2; shift; shift 109 ;; 110 -savecompilers) 111 savecompilers=true; shift 112 ;; 113 -nopristinecache) 114 nopristinecache=true; shift 115 ;; 116 -nogpg) 117 nogpg=true; shift 118 ;; 119 -stop) 120 stop=true; shift 121 ;; 122 -svnpath) 123 svnpath=$2; shift; shift 124 ;; 125 -*) 126 echo "Invalid option: $1" 127 usage 128 ;; 129 *) 130 break 131 ;; 132 esac 133done 134 135test $# -eq 0 && usage 136 137SOURCE=$1 138PATCHES= 139STATE= 140BUILD= 141 142if [[ $# < 2 ]]; then 143 PATCHES=$SOURCE/patches 144else 145 PATCHES=$2 146fi 147if [[ $# < 3 ]]; then 148 STATE=$SOURCE/state 149else 150 STATE=$3 151fi 152if [[ $# < 4 ]]; then 153 BUILD=$SOURCE/obj 154else 155 BUILD=$4 156fi 157 158[ -d $PATCHES ] || makedir $PATCHES 159[ -d $STATE ] || makedir $STATE 160[ -d $STATE/patched ] || makedir $STATE/patched 161[ -d $SOURCE ] || makedir $SOURCE 162[ -f $SOURCE/config.guess ] || { 163 cd $SOURCE 164 svn -q co $svnpath/trunk . 165 if [ $? -ne 0 ]; then 166 echo "ERROR: initial svn checkout failed" 167 exit 1 168 fi 169} 170 171# This can contain required local settings: 172# default_config configure options, always passed 173# default_make make bootstrap options, always passed 174# default_check make check options, always passed 175[ -f $STATE/defaults ] && . $STATE/defaults 176 177VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 178 179exec >> $STATE/tester.log 2>&1 || exit 1 180set -x 181 182TESTING=$STATE/testing 183REPORT=$TESTING/report 184PRISTINE=$TESTING/pristine 185PATCHED=$TESTING/patched 186PATCH= 187TARGET=`$SOURCE/config.guess || exit 1` 188TESTLOGS="gcc/testsuite/gcc/gcc.sum 189gcc/testsuite/gfortran/gfortran.sum 190gcc/testsuite/g++/g++.sum 191gcc/testsuite/objc/objc.sum 192$TARGET/libstdc++-v3/testsuite/libstdc++.sum 193$TARGET/libffi/testsuite/libffi.sum 194$TARGET/libgomp/testsuite/libgomp.sum 195$TARGET/libmudflap/testsuite/libmudflap.sum" 196COMPILERS="gcc/cc1 197gcc/cc1obj 198gcc/cc1plus 199gcc/f951 200gcc/jc1 201gcc/gnat1 202gcc/tree1" 203 204now () { 205 echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"` 206} 207 208report () { 209 echo "$@" >> $REPORT 210} 211 212freport () { 213 if [ -s $1 ]; then 214 report "(cat $1" 215 cat $1 >> $REPORT 216 report "tac)" 217 fi 218} 219 220cleanup () { 221 cd $SOURCE 222 svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v 223} 224 225selfexec () { 226 exec ${CONFIG_SHELL-/bin/sh} $0 $args 227} 228 229update () { 230 svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"` 231 if [ x$svn_branch = x ]; then 232 svn_branch=trunk 233 fi 234 235 svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"` 236 if [ x$svn_revision = x ]; then 237 svn_revision=HEAD 238 fi 239 240 cleanup 241 cd $SOURCE 242 case $svn_branch in 243 trunk) 244 if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then 245 report "failed to update svn sources with" 246 report "svn switch -r $svn_revision $svnpath/trunk" 247 freport $TESTING/svn 248 return 1 249 fi 250 ;; 251 252 ${svnpath}*) 253 if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then 254 report "failed to update svn sources with" 255 report "svn switch -r $svn_revision $svn_branch" 256 freport $TESTING/svn 257 return 1 258 fi 259 ;; 260 261 *) 262 if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then 263 report "failed to update svn sources with" 264 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch" 265 freport $TESTING/svn 266 return 1 267 fi 268 ;; 269 esac 270 contrib/gcc_update --touch 271 272 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 273 if [[ $VERSION < $current_version ]]; then 274 if [ -f $SOURCE/contrib/patch_tester.sh ]; then 275 selfexec 276 fi 277 fi 278 279 return 0 280} 281 282apply_patch () { 283 if [ $nogpg = false ]; then 284 if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then 285 report "your patch failed to verify:" 286 freport $TESTING/gpgverify 287 return 1 288 fi 289 fi 290 291 cd $SOURCE 292 if ! patch -p0 < $PATCH &> $TESTING/patching ; then 293 report "your patch failed to apply:" 294 report "(check that the patch was created at the top level)" 295 freport $TESTING/patching 296 return 1 297 fi 298 299 # Just assume indexes for now -- not really great, but svn always 300 # makes them. 301 grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do 302 # If the patch resulted in an empty file, delete it. 303 # This is how svn reports deletions. 304 if [ ! -s $file ]; then 305 rm -f $file 306 report "Deleting empty file $file" 307 fi 308 done 309} 310 311save_compilers () { 312 for COMPILER in $COMPILERS ; do 313 if [ -f $BUILD/$COMPILER ]; then 314 cp $BUILD/$COMPILER $PRISTINE 315 fi 316 done 317} 318 319bootntest () { 320 rm -rf $BUILD 321 mkdir $BUILD 322 cd $BUILD 323 324 CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"` 325 CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS" 326 if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then 327 report "configure with `basename $1` version failed with:" 328 freport $1/configure 329 return 1 330 fi 331 332 MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"` 333 MAKE_ARGS="$default_make $MAKE_ARGS" 334 if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then 335 report "bootstrap with `basename $1` version failed with last lines:" 336 tail -30 $1/bootstrap > $1/last_bootstrap 337 freport $1/last_bootstrap 338 report "grep --context=20 Error bootstrap:" 339 grep --context=20 Error $1/bootstrap > $1/bootstrap_error 340 freport $1/bootstrap_error 341 return 1 342 fi 343 344 CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"` 345 CHECK_OPTIONS="$default_check $CHECK_OPTIONS" 346 eval make $dashj $CHECK_OPTIONS -k check &> $1/check 347 348 SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`" 349 if [ x$SUITESRUN = x ]; then 350 report "check with `basename $1` version failed, no testsuites were run" 351 return 1 352 fi 353 354 for LOG in $TESTLOGS ; do 355 if [ -f $BUILD/$LOG ]; then 356 mv $BUILD/$LOG $1 357 mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1 358 fi 359 done 360 361 return 0 362} 363 364bootntest_patched () { 365 cleanup 366 mkdir -p $PATCHED 367 apply_patch && bootntest $PATCHED 368 return $? 369} 370 371# Build the pristine tree with exactly the same options as the patch under test. 372bootntest_pristine () { 373 cleanup 374 current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"` 375 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 376 PRISTINE=$STATE/$current_branch/$current_version 377 378 if [ $nopristinecache = true ]; then 379 rm -rf $PRISTINE 380 fi 381 if [ -d $PRISTINE ]; then 382 ln -s $PRISTINE $TESTING/pristine 383 return 0 384 else 385 mkdir -p $PRISTINE 386 ln -s $PRISTINE $TESTING/pristine 387 bootntest $PRISTINE 388 RETVAL=$? 389 if [ $RETVAL = 0 -a $savecompilers = true ]; then 390 save_compilers 391 fi 392 return $RETVAL 393 fi 394} 395 396regtest () { 397 touch $1/report 398 touch $1/passes 399 touch $1/failed 400 touch $1/regress 401 402 for LOG in $TESTLOGS ; do 403 NLOG=`basename $LOG` 404 if [ -f $1/$NLOG ]; then 405 awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG 406 fi 407 done | sort | uniq > $1/failed 408 409 comm -12 $1/failed $1/passes >> $1/regress 410 NUMREGRESS=`wc -l < $1/regress | tr -d ' '` 411 412 if [ $NUMREGRESS -eq 0 ] ; then 413 for LOG in $TESTLOGS ; do 414 NLOG=`basename $LOG` 415 if [ -f $1/$NLOG ] ; then 416 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG 417 fi 418 done | sort | uniq | comm -23 - $1/failed > $1/passes 419 echo "there are no regressions with your patch." >> $1/report 420 else 421 echo "with your patch there are $NUMREGRESS regressions." >> $1/report 422 echo "list of regressions with your patch:" >> $1/report 423 cat $1/regress >> $1/report 424 fi 425} 426 427contrib_compare_tests () { 428 report "comparing logs with contrib/compare_tests:" 429 for LOG in $TESTLOGS ; do 430 NLOG=`basename $LOG` 431 if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then 432 $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG 433 freport $TESTING/compare_$NLOG 434 fi 435 done 436} 437 438compare_passes () { 439 regtest $PRISTINE 440 cp $PRISTINE/passes $PATCHED 441 regtest $PATCHED 442 freport $PATCHED/report 443 report "FAILs with patched version:" 444 freport $PATCHED/failed 445 report "FAILs with pristine version:" 446 freport $PRISTINE/failed 447 448 # contrib_compare_tests 449} 450 451write_report () { 452 backup_patched=$STATE/patched/`now` 453 report "The files used for the validation of your patch are stored in $backup_patched on the tester machine." 454 455 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 456 if [ x$EMAIL != x ]; then 457 mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL 458 fi 459 460 mv $TESTING $backup_patched 461} 462 463announce () { 464 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 465 if [ x$EMAIL != x ]; then 466 467 START_REPORT=$TESTING/start_report 468 echo "Hi, " >> $START_REPORT 469 echo "I'm the automatic tester running on $TARGET." >> $START_REPORT 470 echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT 471 echo "Bye, your automatic tester." >> $START_REPORT 472 mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL 473 fi 474} 475 476# After selfexec, $TESTING is already set up. 477if [ -d $TESTING ]; then 478 # The only file in $TESTING is the patch. 479 PATCH=`ls -rt -1 $TESTING | head -1` 480 PATCH=$TESTING/$PATCH 481 if [ -f $PATCH ]; then 482 bootntest_patched && bootntest_pristine && compare_passes 483 write_report 484 fi 485fi 486 487firstpatch=true 488while true; do 489 PATCH=`ls -rt -1 $PATCHES | head -1` 490 if [ x$PATCH = x ]; then 491 if [ $stop = true ]; then 492 if [ $firstpatch = true ]; then 493 echo "No patches ready to test, quitting." 494 exit 1 495 else 496 echo "No more patches to test." 497 exit 0 498 fi 499 fi 500 sleep ${standby}m 501 else 502 firstpatch=false 503 sysload=`uptime | cut -d, -f 5` 504 if [[ $sysload > $watermark ]]; then 505 # Wait a bit when system load is too high. 506 sleep ${standby}m 507 else 508 mkdir -p $TESTING 509 mv $PATCHES/$PATCH $TESTING/ 510 PATCH=$TESTING/$PATCH 511 512 announce 513 update && bootntest_patched && bootntest_pristine && compare_passes 514 write_report 515 fi 516 fi 517done 518