18a272653SPeter Holm#!/bin/sh 28a272653SPeter Holm 38a272653SPeter Holm# 48a272653SPeter Holm# Copyright (c) 2008-2009, 2012-13 Peter Holm <pho@FreeBSD.org> 58a272653SPeter Holm# All rights reserved. 68a272653SPeter Holm# 78a272653SPeter Holm# Redistribution and use in source and binary forms, with or without 88a272653SPeter Holm# modification, are permitted provided that the following conditions 98a272653SPeter Holm# are met: 108a272653SPeter Holm# 1. Redistributions of source code must retain the above copyright 118a272653SPeter Holm# notice, this list of conditions and the following disclaimer. 128a272653SPeter Holm# 2. Redistributions in binary form must reproduce the above copyright 138a272653SPeter Holm# notice, this list of conditions and the following disclaimer in the 148a272653SPeter Holm# documentation and/or other materials provided with the distribution. 158a272653SPeter Holm# 168a272653SPeter Holm# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 178a272653SPeter Holm# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 188a272653SPeter Holm# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 198a272653SPeter Holm# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 208a272653SPeter Holm# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 218a272653SPeter Holm# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 228a272653SPeter Holm# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 238a272653SPeter Holm# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 248a272653SPeter Holm# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 258a272653SPeter Holm# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 268a272653SPeter Holm# SUCH DAMAGE. 278a272653SPeter Holm# 288a272653SPeter Holm 298a272653SPeter Holm# Run all the scripts in stress2/misc. 308a272653SPeter Holm 318a272653SPeter Holm[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 328a272653SPeter Holm 338a272653SPeter Holm# Log and config files: 348a272653SPeter Holmsdir=/tmp/stress2.d 358a272653SPeter Holmmkdir -p $sdir 368a272653SPeter Holmallconfig=$sdir/`hostname` # config file 378a272653SPeter Holmallfaillog=$sdir/fail # Tests that failed 388a272653SPeter Holmalllast=$sdir/last # Last test run 398a272653SPeter Holmalllist=$sdir/list # -o list 408a272653SPeter Holmalllog=$sdir/log # Tests run 418a272653SPeter Holmalloutput=$sdir/output # Output from current test 428a272653SPeter Holmallexcess=$sdir/excessive # Tests with excessive runtime 438a272653SPeter Holmallelapsed=$sdir/elapsed # Test runtime 448a272653SPeter Holmalllocal=$sdir/all.exclude # Local exclude list 45219d8fb0SPeter Holmexitonerror=0 # -e option 468a272653SPeter Holmloops=0 # Times to run the tests 478a272653SPeter Holm# Get kernel config + revision 488a272653SPeter Holmrev=`uname -a | awk '{print $7}' | sed 's/://'` 498a272653SPeter Holmrev="`uname -a | sed 's#.*/compile/##; s/ .*//'` $rev" 508a272653SPeter Holm 51219d8fb0SPeter Holmargs=`getopt acel:m:no "$@"` 528a272653SPeter Holm[ $? -ne 0 ] && 53219d8fb0SPeter Holm echo "Usage $0 [-a] [-c] [-e] [-l <val>] [-m <min.>] [-n] [-o] [<tests>]" && 548a272653SPeter Holm exit 1 558a272653SPeter Holmset -- $args 568a272653SPeter Holmfor i; do 578a272653SPeter Holm case "$i" in 588a272653SPeter Holm -a) all=1 # Run all tests 598a272653SPeter Holm echo "Note: including known problem tests." 608a272653SPeter Holm shift 618a272653SPeter Holm ;; 628a272653SPeter Holm -c) rm -f $alllast # Clear last know test 638a272653SPeter Holm rm -f $alllist 648a272653SPeter Holm shift 658a272653SPeter Holm ;; 66219d8fb0SPeter Holm -e) exitonerror=1 67219d8fb0SPeter Holm shift 68219d8fb0SPeter Holm ;; 698a272653SPeter Holm -l) loops=$2 # Number of time to run 708a272653SPeter Holm shift; shift 718a272653SPeter Holm ;; 728a272653SPeter Holm -m) minutes=$(($2 * 60)) # Run for minutes 738a272653SPeter Holm shift; shift 748a272653SPeter Holm ;; 758a272653SPeter Holm -n) noshuffle=1 # Do not shuffle the list of tests 768a272653SPeter Holm shift # Resume test after last test 778a272653SPeter Holm ;; 788a272653SPeter Holm -o) loops=1 # Only run once 798a272653SPeter Holm shift 808a272653SPeter Holm ;; 818a272653SPeter Holm --) 828a272653SPeter Holm shift 838a272653SPeter Holm break 848a272653SPeter Holm ;; 858a272653SPeter Holm esac 868a272653SPeter Holmdone 878a272653SPeter Holm 888a272653SPeter Holmexport allconfig 898a272653SPeter Holmif [ ! -f $allconfig ]; then 908a272653SPeter Holm echo "Creating local configuration file: $allconfig." 918a272653SPeter Holm ../tools/setup.sh || exit 1 928a272653SPeter Holmfi 938a272653SPeter Holm 948a272653SPeter Holm. ../default.cfg 958a272653SPeter Holm 968a272653SPeter Holm# Sanity checks 978a272653SPeter Holmminspace=$((1024 * 1024)) # in k 988a272653SPeter Holm[ -d `dirname "$diskimage"` ] || 998a272653SPeter Holm { echo "diskimage dir: $diskimage not found"; exit 1; } 1008a272653SPeter Holm[ `df -k $(dirname $diskimage) | tail -1 | awk '{print $4}'` -lt \ 1018a272653SPeter Holm $minspace ] && 1028a272653SPeter Holm echo "Warn: Not enough disk space on `dirname $diskimage` " \ 1038a272653SPeter Holm "for \$diskimage" 1048a272653SPeter Holm[ ! -d $(dirname $RUNDIR) ] && 1058a272653SPeter Holm echo "No such \$RUNDIR \"`dirname $RUNDIR`\"" && 1068a272653SPeter Holm exit 1 1078a272653SPeter Holm[ `sysctl -n hw.physmem` -le $((3 * 1024 * 1024 * 1024)) ] && 1088a272653SPeter Holm echo "Warn: Small RAM size for stress tests `sysctl -n hw.physmem`" 1098a272653SPeter Holm[ `df -k $(dirname $RUNDIR) | tail -1 | awk '{print $4}'` -lt \ 1108a272653SPeter Holm $minspace ] && 1118a272653SPeter Holm echo "Warn: Not enough disk space on `dirname $RUNDIR` for \$RUNDIR" 1128a272653SPeter Holmid $testuser > /dev/null 2>&1 || 1138a272653SPeter Holm { echo "\$testuser \"$testuser\" not found."; exit 1; } 1148a272653SPeter Holmprobe=`dirname $RUNDIR`/probe 1158a272653SPeter Holmsu $testuser -c "touch $probe" > /dev/null 2>&1 1168a272653SPeter Holm[ -f $probe ] && rm $probe || 1178a272653SPeter Holm { echo "No write access to `dirname $RUNDIR`."; exit 1; } 1188a272653SPeter Holm[ `swapinfo | wc -l` -eq 1 ] && 1198a272653SPeter Holm echo "Consider adding a swap disk. Many tests rely on this." 1208a272653SPeter Holmmount | grep -wq $mntpoint && 1218a272653SPeter Holm echo "\$mntpoint ($mntpoint) is already in use" && exit 1 1228a272653SPeter Holm[ -x ../testcases/run/run ] || 1238a272653SPeter Holm (cd ..; make) 1248a272653SPeter Holmping -c 2 -t 2 $BLASTHOST > /dev/null 2>&1 || 1258a272653SPeter Holm { echo "Note: Can not ping \$BLASTHOST: $BLASTHOST"; } 1268a272653SPeter Holmecho "$loops" | grep -Eq "^[0-9]+$" || 1278a272653SPeter Holm { echo "The -l argument must be a positive number"; exit 1; } 1288a272653SPeter Holm[ `grep "^[a-zA-Z].*\.sh" $alllocal 2>/dev/null | wc -l` -ne 0 ] && 1298a272653SPeter Holm echo "Using $alllocal" 1308a272653SPeter Holm 1318a272653SPeter Holmfind `dirname $alllast` -maxdepth 1 -name $alllast -mtime +12h -delete 1328a272653SPeter Holmtouch $alllast $alllog 1338a272653SPeter Holmchmod 640 $alllast $alllog 1348a272653SPeter Holmfind ../testcases -perm -1 \( -name "*.debug" -o -name "*.full" \) -delete 1358a272653SPeter Holmtail -2000 $alllog > ${alllog}.new; mv ${alllog}.new $alllog 1368a272653SPeter Holmtouch $allelapsed 1378a272653SPeter Holmtail -20000 $allelapsed > ${allelapsed}.new; mv ${allelapsed}.new $allelapsed 1388a272653SPeter Holm 1398a272653SPeter Holmconsole=/dev/console 1408a272653SPeter Holmprintf "\r\n" > $console & 1418a272653SPeter Holmpid=$! 1428a272653SPeter Holmsleep 1 1438a272653SPeter Holmkill -0 $pid > /dev/null 2>&1 && 1448a272653SPeter Holm{ console=/dev/null; kill -9 $pid; } 1458a272653SPeter Holmwhile pgrep -q fsck; do sleep 10; done 1468a272653SPeter Holm 1478a272653SPeter Holmstatus() { 1488a272653SPeter Holm local s2 r 1498a272653SPeter Holm 1508a272653SPeter Holm s2=`date +%s` 1518a272653SPeter Holm r=$(echo "elapsed $(((s2 - s1) / 86400)) day(s)," \ 1528a272653SPeter Holm "`date -u -j -f '%s' '+%H:%M.%S' $((s2 - s1))`") 1538a272653SPeter Holm printf "`date '+%Y%m%d %T'` all.sh done, $r\n" 1548a272653SPeter Holm printf "`date '+%Y%m%d %T'` all.sh done, $r\r\n" > $console 1558a272653SPeter Holm} 1568a272653SPeter Holm 1578a272653SPeter Holmintr() { 1588a272653SPeter Holm printf "\nExit all.sh\n" 1598a272653SPeter Holm ./cleanup.sh 1608a272653SPeter Holm exit 1 1618a272653SPeter Holm} 1628a272653SPeter Holmtrap status EXIT 1638a272653SPeter Holmtrap intr INT 1648a272653SPeter Holm 1658a272653SPeter Holm[ -f all.debug.inc ] && . all.debug.inc 1668a272653SPeter Holms1=`date +%s` 167219d8fb0SPeter Holmtouch $sdir/starttime 1688a272653SPeter Holmwhile true; do 1698a272653SPeter Holm exclude=`cat all.exclude $alllocal 2>/dev/null | sed '/^#/d' | 1708a272653SPeter Holm grep "^[a-zA-Z].*\.sh" | awk '{print $1}'` 1718a272653SPeter Holm list=`echo *.sh` 1728a272653SPeter Holm [ $# -ne 0 ] && list=$* 1738a272653SPeter Holm list=`echo $list | 1748a272653SPeter Holm sed "s/[[:<:]]all\.sh[[:>:]]//g;\ 1758a272653SPeter Holm s/[[:<:]]cleanup\.sh[[:>:]]//g"` 1768a272653SPeter Holm 1778a272653SPeter Holm if [ -n "$noshuffle" -a $# -eq 0 ]; then 1788a272653SPeter Holm last=`cat $alllast` 1798a272653SPeter Holm if [ -n "$last" ]; then 1808a272653SPeter Holm last=`basename $last` 1818a272653SPeter Holm l=`cat "$alllist" | sed "s/.*$last//"` 1828a272653SPeter Holm [ -z "$l" ] && l=$list # start over 1838a272653SPeter Holm list=$l 1848a272653SPeter Holm echo "Last test was $last,"\ 1858a272653SPeter Holm "resuming test at" \ 1868a272653SPeter Holm "`echo "$list" | awk '{print $1}'`" 1878a272653SPeter Holm fi 1888a272653SPeter Holm fi 1898a272653SPeter Holm [ -n "$noshuffle" ] || 1908a272653SPeter Holm list=`echo $list | tr ' ' '\n' | sort -R | 1918a272653SPeter Holm tr '\n' ' '` 1928a272653SPeter Holm 1938a272653SPeter Holm lst="" 1948a272653SPeter Holm for i in $list; do 1958a272653SPeter Holm [ -z "$all" ] && echo $exclude | grep -qw `basename $i` && 1968a272653SPeter Holm continue 1978a272653SPeter Holm lst="$lst $i" 1988a272653SPeter Holm done 1998a272653SPeter Holm [ -z "$lst" ] && exit 2008a272653SPeter Holm echo "$lst" > $alllist 2018a272653SPeter Holm 2028a272653SPeter Holm pgrep -fq vmstat.sh || 2038a272653SPeter Holm daemon ../tools/vmstat.sh > /tmp/stress2.d/vmstat 2>&1 2048a272653SPeter Holm 2058a272653SPeter Holm n1=0 2068a272653SPeter Holm n2=`echo $lst | wc -w | sed 's/ //g'` 2078a272653SPeter Holm for i in $lst; do 2088a272653SPeter Holm i=`basename $i` 2098a272653SPeter Holm [ ! -f ./$i ] && { echo "No such file ./$i"; continue; } 2108a272653SPeter Holm n1=$((n1 + 1)) 2118a272653SPeter Holm echo $i > $alllast 2128a272653SPeter Holm ./cleanup.sh || exit 1 2138a272653SPeter Holm ts=`date '+%Y%m%d %T'` 2148a272653SPeter Holm echo "$ts all: $i" 2158a272653SPeter Holm printf "$ts all ($n1/$n2): $i\n" >> $alllog 2168a272653SPeter Holm printf "$ts all ($n1/$n2): $i\r\n" > $console 2178a272653SPeter Holm logger "Starting stress2 test all.sh: $i" 2188a272653SPeter Holm [ $all_debug ] && pre_debug 2198a272653SPeter Holm [ -f $i ] || loops=1 # break 2208a272653SPeter Holm sync; sleep .5; sync; sleep .5 2218a272653SPeter Holm grep -E "^USE_TIMEOUT=1" $i && TIMEOUT_ONE=1 || 2228a272653SPeter Holm unset TIMEOUT_ONE 2238a272653SPeter Holm start=`date '+%s'` 2248a272653SPeter Holm ( 2258a272653SPeter Holm if [ $USE_TIMEOUT ] || [ $TIMEOUT_ONE ]; then 2268a272653SPeter Holm timeout -k 1m 1h ./$i 2278a272653SPeter Holm else 2288a272653SPeter Holm ./$i 2298a272653SPeter Holm fi 2308a272653SPeter Holm e=$? 2318a272653SPeter Holm [ $e -ne 0 ] && 2328a272653SPeter Holm echo "FAIL $i exit code $e" 2338a272653SPeter Holm ) 2>&1 | tee $alloutput 234219d8fb0SPeter Holm grep -qw FAIL $alloutput && e=1 || e=0 2358a272653SPeter Holm ts=`date '+%Y%m%d %T'` 2368a272653SPeter Holm grep -qw FAIL $alloutput && 2378a272653SPeter Holm echo "$ts $rev $i" >> $allfaillog && 2388a272653SPeter Holm logger "stress2 test $i failed" 2398a272653SPeter Holm grep -qw FATAL $alloutput && exit $e 2408a272653SPeter Holm rm -f $alloutput 2418a272653SPeter Holm printf "$ts $rev $i $((`date '+%s'` - start))\n" >> \ 2428a272653SPeter Holm $allelapsed 2438a272653SPeter Holm [ -f ../tools/ministat.sh ] && 2448a272653SPeter Holm ../tools/ministat.sh $allelapsed $i 2458a272653SPeter Holm [ $((`date '+%s'` - start)) -gt 1980 ] && 2460d30f3afSPeter Holm printf "$ts $rev $i %d min\n" \ 2470d30f3afSPeter Holm $(((`date '+%s'` - start) / 60)) >> $allexcess 2488a272653SPeter Holm while pgrep -q "^swap$"; do 2498a272653SPeter Holm echo "swap still running" 2508a272653SPeter Holm sleep 2 2518a272653SPeter Holm done 2528a272653SPeter Holm [ $USE_SWAPOFF ] && { swapoff -a; swapon -a; } 2538a272653SPeter Holm [ $all_debug ] && post_debug 2548a272653SPeter Holm [ $minutes ] && [ $((`date +%s` - s1)) -ge $minutes ] && 2558a272653SPeter Holm break 2 256219d8fb0SPeter Holm [ $exitonerror -eq 1 -a $e -ne 0 ] && break 2 2578a272653SPeter Holm done 2588a272653SPeter Holm [ $((loops -= 1)) -eq 0 ] && break 2598a272653SPeter Holmdone 2608a272653SPeter Holm[ -x ../tools/fail.sh ] && ../tools/fail.sh 2618a272653SPeter Holmfind /tmp . -name "*.core" -mtime -2 -maxdepth 2 -ls 2>/dev/null 262