1#! /bin/sh
2# Copyright (C) 2011-2021 Free Software Foundation, Inc.
3#
4# This program is free software; you can redistribute it and/or modify
5# it under the terms of the GNU General Public License as published by
6# the Free Software Foundation; either version 2, or (at your option)
7# any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17# Test the 'recheck' target for TAP test protocol.
18# Keep in sync with 'test-driver-custom-multitest-recheck.sh'.
19
20. test-init.sh
21
22fetch_tap_driver
23
24cat >> configure.ac <<END
25AC_OUTPUT
26END
27
28cat > Makefile.am << 'END'
29TEST_LOG_DRIVER = $(srcdir)/tap-driver
30TESTS = a.test b.test c.test d.test
31END
32
33cat > a.test << 'END'
34#! /bin/sh
35echo 1..2
36echo ok 1
37echo ok 2
38: > a.run
39END
40
41cat > b.test << 'END'
42#! /bin/sh
43: > b.run
44echo 1..2
45echo ok 1
46if test -f b.ok; then
47  echo ok 2
48else
49  echo 'Bail out!'
50fi
51END
52
53cat > c.test << 'END'
54#! /bin/sh
55echo 1..3
56if test -f c.pass; then
57  echo 'ok - c is ok :-)'
58else
59  echo 'not ok - c is ko :-('
60fi
61if test -f c.xfail; then
62  echo 'not ok 2 # TODO'
63else
64  echo 'ok 2 # TODO'
65fi
66echo 'not ok 3 - blah blah # TODO need better diagnostic'
67: > c.run
68END
69
70cat > d.test << 'END'
71#! /bin/sh
72test -f ./d.count && . ./d.count
73echo 1..${test_count-2}
74echo ok 1 '# SKIP: who cares ...'
75(. ./d.extra) || echo 'not ok 2 - d.extra failed'
76: > d.run
77END
78
79chmod a+x *.test
80
81$ACLOCAL
82$AUTOCONF
83$AUTOMAKE
84
85do_recheck ()
86{
87  case $* in
88    --fail) status=FAIL;;
89    --pass) status=0;;
90         *) fatal_ "invalid usage of function 'do_recheck'";;
91  esac
92  rm -f *.run
93  run_make -O -e $status recheck || { ls -l; exit 1; }
94  ls -l
95}
96
97for vpath in : false; do
98  if $vpath; then
99    mkdir build
100    cd build
101    srcdir=..
102  else
103    srcdir=.
104  fi
105
106  $srcdir/configure
107
108  : A "make recheck" in a clean tree should run no tests.
109  do_recheck --pass
110  cat test-suite.log
111  test ! -e a.run
112  test ! -e a.log
113  test ! -e b.run
114  test ! -e b.log
115  test ! -e c.run
116  test ! -e c.log
117  test ! -e d.run
118  test ! -e d.log
119  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
120
121  : Run the tests for the first time.
122  run_make -O -e FAIL check
123  ls -l
124  # All the test scripts should have run.
125  test -f a.run
126  test -f b.run
127  test -f c.run
128  test -f d.run
129  count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
130
131  : Let us make b.test pass.
132  using_gmake || $sleep # Required by BSD make.
133  echo OK > b.ok
134  do_recheck --fail
135  # a.test has been successful the first time, so no need to re-run it.
136  # Similar considerations apply to similar checks, below.
137  test ! -e a.run
138  test -f b.run
139  test -f c.run
140  test -f d.run
141  count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
142
143  : Let us make the first part of c.test pass.
144  using_gmake || $sleep # Required by BSD make.
145  echo OK > c.pass
146  do_recheck --fail
147  test ! -e a.run
148  test ! -e b.run
149  test -f c.run
150  test -f d.run
151  count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
152
153  : Let us make also the second part of c.test pass.
154  using_gmake || $sleep # Required by BSD make.
155  echo KO > c.xfail
156  do_recheck --fail
157  test ! -e a.run
158  test ! -e b.run
159  test -f c.run
160  test -f d.run
161  count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
162
163  : Nothing changed, so only d.test should be run.
164  for i in 1 2; do
165    using_gmake || $sleep # Required by BSD make.
166    do_recheck --fail
167    test ! -e a.run
168    test ! -e b.run
169    test ! -e c.run
170    test -f d.run
171    count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
172  done
173
174  : Let us make d.test run more testcases, and experience _more_ failures.
175  using_gmake || $sleep # Required by BSD make.
176  echo 'test_count=9' > d.count
177  unindent > d.extra <<'END'
178    echo 'ok # SKIP s'
179    echo 'not ok - f 1'
180    echo 'ok - p 1'
181    echo 'not ok - f 2'
182    echo 'ok # TODO xp'
183    echo 'not ok - f 3'
184    echo 'not ok - f 4'
185    echo 'ok - p 2'
186    echo 'ok' # Extra test.
187    echo 'Bail out!'
188END
189  do_recheck --fail
190  test ! -e a.run
191  test ! -e b.run
192  test ! -e c.run
193  test -f d.run
194  count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
195
196  : Let us finally make d.test pass.
197  using_gmake || $sleep # Required by BSD make.
198  echo 'test_count=1' > d.count
199  echo : > d.extra
200  do_recheck --pass
201  test ! -e a.run
202  test ! -e b.run
203  test ! -e c.run
204  test -f d.run
205  count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
206
207  : All tests have been successful or skipped, nothing should be re-run.
208  using_gmake || $sleep # Required by BSD make.
209  do_recheck --pass
210  test ! -e a.run
211  test ! -e b.run
212  test ! -e c.run
213  test ! -e d.run
214  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
215
216  cd $srcdir
217
218done
219
220:
221