xref: /freebsd/contrib/kyua/bootstrap/testsuite.at (revision 1f474190) 
1dnl Copyright 2010 The Kyua Authors.
2dnl All rights reserved.
3dnl
4dnl Redistribution and use in source and binary forms, with or without
5dnl modification, are permitted provided that the following conditions are
6dnl met:
7dnl
8dnl * Redistributions of source code must retain the above copyright
9dnl   notice, this list of conditions and the following disclaimer.
10dnl * Redistributions in binary form must reproduce the above copyright
11dnl   notice, this list of conditions and the following disclaimer in the
12dnl   documentation and/or other materials provided with the distribution.
13dnl * Neither the name of Google Inc. nor the names of its contributors
14dnl   may be used to endorse or promote products derived from this software
15dnl   without specific prior written permission.
16dnl
17dnl THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18dnl "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19dnl LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20dnl A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21dnl OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22dnl SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23dnl LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24dnl DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25dnl THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26dnl (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27dnl OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29AT_INIT([bootstrapping tests])
30
31
32m4_define([GUESS_TOPDIR], {
33    old=$(pwd)
34    cd "$(dirname ${as_myself})"
35    # We need to locate a build product, not a source file, because the
36    # test suite may be run outside of the source tree (think distcheck).
37    while test $(pwd) != '/' -a ! -e bootstrap/plain_helpers; do
38        cd ..
39    done
40    topdir=$(pwd)
41    cd ${old}
42    echo ${topdir}
43})
44
45
46m4_define([CREATE_ATF_HELPERS], [
47    AT_DATA([Kyuafile], [
48syntax(2)
49test_suite("bootstrap")
50atf_test_program{name="atf_helpers"}
51])
52    ln -s $(GUESS_TOPDIR)/bootstrap/atf_helpers atf_helpers
53])
54m4_define([RUN_ATF_HELPERS],
55    [HOME=$(pwd) TESTS="$1" kyua --config=none \
56        test --results-file=bootstrap.db $2])
57
58
59m4_define([CREATE_PLAIN_HELPERS], [
60    AT_DATA([Kyuafile], [
61syntax(2)
62test_suite("bootstrap")
63atf_test_program{name="plain_helpers"}
64])
65    ln -s $(GUESS_TOPDIR)/bootstrap/plain_helpers plain_helpers
66])
67m4_define([RUN_PLAIN_HELPER],
68    [HOME=$(pwd) HELPER="$1" kyua --config=none \
69        test --results-file=bootstrap.db])
70
71
72AT_SETUP([test program crashes in test list])
73AT_TESTED([kyua])
74
75CREATE_PLAIN_HELPERS
76AT_CHECK([RUN_PLAIN_HELPER([abort_test_cases_list])], [1], [stdout], [])
77re='plain_helpers:__test_cases_list__.*broken.*Test program received signal'
78AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
79
80AT_CLEANUP
81
82
83AT_SETUP([test program prints an empty test list])
84AT_TESTED([kyua])
85
86CREATE_PLAIN_HELPERS
87AT_CHECK([RUN_PLAIN_HELPER([empty_test_cases_list])], [1], [stdout], [])
88re="plain_helpers:__test_cases_list__.*broken.*Invalid header.*got ''"
89AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
90
91AT_CLEANUP
92
93
94AT_SETUP([test program with zero test cases])
95AT_TESTED([kyua])
96
97CREATE_PLAIN_HELPERS
98AT_CHECK([RUN_PLAIN_HELPER([zero_test_cases])], [1], [stdout], [])
99re='plain_helpers:__test_cases_list__.*broken.*No test cases'
100AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
101
102AT_CLEANUP
103
104
105AT_SETUP([run test case that passes])
106AT_TESTED([kyua])
107
108CREATE_ATF_HELPERS
109AT_CHECK([RUN_ATF_HELPERS([passes])], [0], [stdout], [])
110AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
111AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
112AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
113
114AT_CLEANUP
115
116
117AT_SETUP([run test case that fails])
118AT_TESTED([kyua])
119
120CREATE_ATF_HELPERS
121AT_CHECK([RUN_ATF_HELPERS([fails])], [1], [stdout], [])
122AT_CHECK([grep "atf_helpers:fails.*failed.*Failed on purpose" stdout],
123         [0], [ignore], [])
124AT_CHECK([grep "atf_helpers:passes" stdout], [1], [], [])
125AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
126
127AT_CLEANUP
128
129
130AT_SETUP([run test case that skips])
131AT_TESTED([kyua])
132
133CREATE_ATF_HELPERS
134AT_CHECK([RUN_ATF_HELPERS([skips])], [0], [stdout], [])
135AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
136AT_CHECK([grep "atf_helpers:passes" stdout], [1], [], [])
137AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
138         [0], [ignore], [])
139
140AT_CLEANUP
141
142
143AT_SETUP([run two test cases, success])
144AT_TESTED([kyua])
145
146CREATE_ATF_HELPERS
147AT_CHECK([RUN_ATF_HELPERS([passes skips])], [0], [stdout], [])
148AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
149AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
150AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
151         [0], [ignore], [])
152
153AT_CLEANUP
154
155
156AT_SETUP([run two test cases, failure])
157AT_TESTED([kyua])
158
159CREATE_ATF_HELPERS
160AT_CHECK([RUN_ATF_HELPERS([fails passes])], [1], [stdout], [])
161AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
162         [1], [], [])
163AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
164AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
165
166AT_CLEANUP
167
168
169AT_SETUP([run mixed test cases])
170AT_TESTED([kyua])
171
172CREATE_ATF_HELPERS
173AT_CHECK([RUN_ATF_HELPERS([fails passes skips])], [1], [stdout], [])
174AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
175         [1], [], [])
176AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
177AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
178         [0], [ignore], [])
179
180AT_CLEANUP
181
182
183AT_SETUP([run tests from build directories])
184AT_TESTED([kyua])
185
186CREATE_ATF_HELPERS
187AT_CHECK([mkdir src], [0], [], [])
188AT_CHECK([mv Kyuafile src], [0], [], [])
189AT_CHECK([mkdir obj], [0], [], [])
190AT_CHECK([mv atf_helpers obj], [0], [], [])
191AT_CHECK([RUN_ATF_HELPERS([fails passes skips],
192                          [--kyuafile=src/Kyuafile --build-root=obj])],
193         [1], [stdout], [])
194AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
195         [1], [], [])
196AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
197AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
198         [0], [ignore], [])
199
200AT_CLEANUP
201