1#! /bin/sh
2# Copyright (C) 2011-2021 Free Software Foundation, Inc.
3#
4# This program is free software; you can redistribute it and/or modify
5# it under the terms of the GNU General Public License as published by
6# the Free Software Foundation; either version 2, or (at your option)
7# any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17# Custom test drivers: try the "recheck" functionality with test protocols
18# that allow multiple testcases in a single test script.  This test not
19# only checks implementation details in Automake's custom test drivers
20# support, but also serves as a "usability test" for our APIs.
21# See also related tests 'test-driver-custom-multitest-recheck2.sh'
22# and 'parallel-tests-recheck-override.sh'.
23# Keep in sync with 'tap-recheck.sh'.
24
25. test-init.sh
26
27cp "$am_testaux_srcdir"/trivial-test-driver . \
28  || fatal_ "failed to fetch auxiliary script trivial-test-driver"
29
30cat >> configure.ac << 'END'
31AC_OUTPUT
32END
33
34cat > Makefile.am << 'END'
35TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
36TESTS = a.test b.test c.test d.test
37END
38
39cat > a.test << 'END'
40#! /bin/sh
41echo PASS: aa
42echo PASS: AA
43: > a.run
44END
45
46cat > b.test << 'END'
47#! /bin/sh
48echo PASS:
49if test -f b.ok; then
50  echo PASS:
51else
52  echo ERROR:
53fi
54: > b.run
55END
56
57cat > c.test << 'END'
58#! /bin/sh
59if test -f c.pass; then
60  echo PASS: c0
61else
62  echo FAIL: c0
63fi
64if test -f c.xfail; then
65  echo XFAIL: c1
66else
67  echo XPASS: c1
68fi
69echo XFAIL: c2
70: > c.run
71END
72
73cat > d.test << 'END'
74#! /bin/sh
75echo SKIP: who cares ...
76(. ./d.extra) || echo FAIL: d.extra failed
77: > d.run
78END
79
80chmod a+x *.test
81
82$ACLOCAL
83$AUTOCONF
84$AUTOMAKE
85
86do_recheck ()
87{
88  case $* in
89    --fail) status=FAIL;;
90    --pass) status=0;;
91         *) fatal_ "invalid usage of function 'do_recheck'";;
92  esac
93  rm -f *.run
94  run_make -O -e $status recheck || { ls -l; exit 1; }
95  ls -l
96}
97
98for vpath in : false; do
99  if $vpath; then
100    mkdir build
101    cd build
102    srcdir=..
103  else
104    srcdir=.
105  fi
106
107  $srcdir/configure
108
109  : A "make recheck" in a clean tree should run no tests.
110  using_gmake || $sleep # Required by BSD make.
111  do_recheck --pass
112  cat test-suite.log
113  test ! -e a.run
114  test ! -e a.log
115  test ! -e b.run
116  test ! -e b.log
117  test ! -e c.run
118  test ! -e c.log
119  test ! -e d.run
120  test ! -e d.log
121  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
122
123  : Run the tests for the first time.
124  run_make -O -e FAIL check
125  ls -l
126  # All the test scripts should have run.
127  test -f a.run
128  test -f b.run
129  test -f c.run
130  test -f d.run
131  count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
132
133  : Let us make b.test pass.
134  using_gmake || $sleep # Required by BSD make.
135  echo OK > b.ok
136  do_recheck --fail
137  # a.test has been successful the first time, so no need to re-run it.
138  # Similar considerations apply to similar checks, below.
139  test ! -e a.run
140  test -f b.run
141  test -f c.run
142  test -f d.run
143  count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
144
145  : Let us make the first part of c.test pass.
146  using_gmake || $sleep # Required by BSD make.
147  echo OK > c.pass
148  do_recheck --fail
149  test ! -e a.run
150  test ! -e b.run
151  test -f c.run
152  test -f d.run
153  count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
154
155  : Let us make also the second part of c.test pass.
156  using_gmake || $sleep # Required by BSD make.
157  echo KO > c.xfail
158  do_recheck --fail
159  test ! -e a.run
160  test ! -e b.run
161  test -f c.run
162  test -f d.run
163  count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
164
165  : Nothing changed, so only d.test should be run.
166  for i in 1 2; do
167    using_gmake || $sleep # Required by BSD make.
168    do_recheck --fail
169    test ! -e a.run
170    test ! -e b.run
171    test ! -e c.run
172    test -f d.run
173    count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
174  done
175
176  : Let us make d.test run more testcases, and experience _more_ failures.
177  using_gmake || $sleep # Required by BSD make.
178  unindent > d.extra <<'END'
179    echo SKIP: s
180    echo FAIL: f 1
181    echo PASS: p 1
182    echo FAIL: f 2
183    echo XPASS: xp
184    echo FAIL: f 3
185    echo FAIL: f 4
186    echo ERROR: e 1
187    echo PASS: p 2
188    echo ERROR: e 2
189END
190  do_recheck --fail
191  test ! -e a.run
192  test ! -e b.run
193  test ! -e c.run
194  test -f d.run
195  count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
196
197  : Let us finally make d.test pass.
198  using_gmake || $sleep # Required by BSD make.
199  echo : > d.extra
200  do_recheck --pass
201  test ! -e a.run
202  test ! -e b.run
203  test ! -e c.run
204  test -f d.run
205  count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
206
207  : All tests have been successful or skipped, nothing should be re-run.
208  using_gmake || $sleep # Required by BSD make.
209  do_recheck --pass
210  test ! -e a.run
211  test ! -e b.run
212  test ! -e c.run
213  test ! -e d.run
214  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
215
216  cd $srcdir
217
218done
219
220:
221