1#! /bin/sh 2# Copyright (C) 2011-2021 Free Software Foundation, Inc. 3# 4# This program is free software; you can redistribute it and/or modify 5# it under the terms of the GNU General Public License as published by 6# the Free Software Foundation; either version 2, or (at your option) 7# any later version. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License 15# along with this program. If not, see <https://www.gnu.org/licenses/>. 16 17# Custom test drivers: try the "recheck" functionality with test protocols 18# that allow multiple testcases in a single test script. This test not 19# only checks implementation details in Automake's custom test drivers 20# support, but also serves as a "usability test" for our APIs. 21# See also related tests 'test-driver-custom-multitest-recheck2.sh' 22# and 'parallel-tests-recheck-override.sh'. 23# Keep in sync with 'tap-recheck.sh'. 24 25. test-init.sh 26 27cp "$am_testaux_srcdir"/trivial-test-driver . \ 28 || fatal_ "failed to fetch auxiliary script trivial-test-driver" 29 30cat >> configure.ac << 'END' 31AC_OUTPUT 32END 33 34cat > Makefile.am << 'END' 35TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver 36TESTS = a.test b.test c.test d.test 37END 38 39cat > a.test << 'END' 40#! /bin/sh 41echo PASS: aa 42echo PASS: AA 43: > a.run 44END 45 46cat > b.test << 'END' 47#! /bin/sh 48echo PASS: 49if test -f b.ok; then 50 echo PASS: 51else 52 echo ERROR: 53fi 54: > b.run 55END 56 57cat > c.test << 'END' 58#! /bin/sh 59if test -f c.pass; then 60 echo PASS: c0 61else 62 echo FAIL: c0 63fi 64if test -f c.xfail; then 65 echo XFAIL: c1 66else 67 echo XPASS: c1 68fi 69echo XFAIL: c2 70: > c.run 71END 72 73cat > d.test << 'END' 74#! /bin/sh 75echo SKIP: who cares ... 76(. ./d.extra) || echo FAIL: d.extra failed 77: > d.run 78END 79 80chmod a+x *.test 81 82$ACLOCAL 83$AUTOCONF 84$AUTOMAKE 85 86do_recheck () 87{ 88 case $* in 89 --fail) status=FAIL;; 90 --pass) status=0;; 91 *) fatal_ "invalid usage of function 'do_recheck'";; 92 esac 93 rm -f *.run 94 run_make -O -e $status recheck || { ls -l; exit 1; } 95 ls -l 96} 97 98for vpath in : false; do 99 if $vpath; then 100 mkdir build 101 cd build 102 srcdir=.. 103 else 104 srcdir=. 105 fi 106 107 $srcdir/configure 108 109 : A "make recheck" in a clean tree should run no tests. 110 using_gmake || $sleep # Required by BSD make. 111 do_recheck --pass 112 cat test-suite.log 113 test ! -e a.run 114 test ! -e a.log 115 test ! -e b.run 116 test ! -e b.log 117 test ! -e c.run 118 test ! -e c.log 119 test ! -e d.run 120 test ! -e d.log 121 count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0 122 123 : Run the tests for the first time. 124 run_make -O -e FAIL check 125 ls -l 126 # All the test scripts should have run. 127 test -f a.run 128 test -f b.run 129 test -f c.run 130 test -f d.run 131 count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1 132 133 : Let us make b.test pass. 134 using_gmake || $sleep # Required by BSD make. 135 echo OK > b.ok 136 do_recheck --fail 137 # a.test has been successful the first time, so no need to re-run it. 138 # Similar considerations apply to similar checks, below. 139 test ! -e a.run 140 test -f b.run 141 test -f c.run 142 test -f d.run 143 count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0 144 145 : Let us make the first part of c.test pass. 146 using_gmake || $sleep # Required by BSD make. 147 echo OK > c.pass 148 do_recheck --fail 149 test ! -e a.run 150 test ! -e b.run 151 test -f c.run 152 test -f d.run 153 count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0 154 155 : Let us make also the second part of c.test pass. 156 using_gmake || $sleep # Required by BSD make. 157 echo KO > c.xfail 158 do_recheck --fail 159 test ! -e a.run 160 test ! -e b.run 161 test -f c.run 162 test -f d.run 163 count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0 164 165 : Nothing changed, so only d.test should be run. 166 for i in 1 2; do 167 using_gmake || $sleep # Required by BSD make. 168 do_recheck --fail 169 test ! -e a.run 170 test ! -e b.run 171 test ! -e c.run 172 test -f d.run 173 count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0 174 done 175 176 : Let us make d.test run more testcases, and experience _more_ failures. 177 using_gmake || $sleep # Required by BSD make. 178 unindent > d.extra <<'END' 179 echo SKIP: s 180 echo FAIL: f 1 181 echo PASS: p 1 182 echo FAIL: f 2 183 echo XPASS: xp 184 echo FAIL: f 3 185 echo FAIL: f 4 186 echo ERROR: e 1 187 echo PASS: p 2 188 echo ERROR: e 2 189END 190 do_recheck --fail 191 test ! -e a.run 192 test ! -e b.run 193 test ! -e c.run 194 test -f d.run 195 count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2 196 197 : Let us finally make d.test pass. 198 using_gmake || $sleep # Required by BSD make. 199 echo : > d.extra 200 do_recheck --pass 201 test ! -e a.run 202 test ! -e b.run 203 test ! -e c.run 204 test -f d.run 205 count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0 206 207 : All tests have been successful or skipped, nothing should be re-run. 208 using_gmake || $sleep # Required by BSD make. 209 do_recheck --pass 210 test ! -e a.run 211 test ! -e b.run 212 test ! -e c.run 213 test ! -e d.run 214 count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0 215 216 cd $srcdir 217 218done 219 220: 221