1# Copyright 2012 The Kyua Authors.
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8# * Redistributions of source code must retain the above copyright
9#   notice, this list of conditions and the following disclaimer.
10# * Redistributions in binary form must reproduce the above copyright
11#   notice, this list of conditions and the following disclaimer in the
12#   documentation and/or other materials provided with the distribution.
13# * Neither the name of Google Inc. nor the names of its contributors
14#   may be used to endorse or promote products derived from this software
15#   without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30# Executes a mock test suite to generate data in the database.
31#
32# \param mock_env The value to store in a MOCK variable in the environment.
33#     Use this to be able to differentiate executions by inspecting the
34#     context of the output.
35# \param dbfile_name File to which to write the path to the generated database
36#     file.
37run_tests() {
38    local mock_env="${1}"; shift
39    local dbfile_name="${1}"; shift
40
41    cat >Kyuafile <<EOF
42syntax(2)
43test_suite("integration")
44atf_test_program{name="simple_all_pass"}
45atf_test_program{name="simple_some_fail"}
46atf_test_program{name="metadata"}
47EOF
48
49    utils_cp_helper simple_all_pass .
50    utils_cp_helper simple_some_fail .
51    utils_cp_helper metadata .
52    atf_check -s exit:1 -o save:stdout -e empty env MOCK="${mock_env}" kyua test
53    grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
54    rm stdout
55
56    # Ensure the results of 'report-html' come from the database.
57    rm Kyuafile simple_all_pass simple_some_fail metadata
58}
59
60
61# Ensure a file has a set of strings.
62#
63# \param file The name of the file to check.
64# \param ... List of strings to check.
65check_in_file() {
66    local file="${1}"; shift
67
68    while [ ${#} -gt 0 ]; do
69        echo "Checking for presence of '${1}' in ${file}"
70        if grep "${1}" "${file}" >/dev/null; then
71            :
72        else
73            atf_fail "Test case output not found in HTML page ${file}"
74        fi
75        shift
76    done
77}
78
79
80# Ensure a file does not have a set of strings.
81#
82# \param file The name of the file to check.
83# \param ... List of strings to check.
84check_not_in_file() {
85    local file="${1}"; shift
86
87    while [ ${#} -gt 0 ]; do
88        echo "Checking for lack of '${1}' in ${file}"
89        if grep "${1}" "${file}" >/dev/null; then
90            atf_fail "Spurious test case output found in HTML page"
91        fi
92        shift
93    done
94}
95
96
97utils_test_case default_behavior__ok
98default_behavior__ok_body() {
99    run_tests "mock1" unused_dbfile_name
100
101    atf_check -s exit:0 -o ignore -e empty kyua report-html
102    for f in \
103        html/index.html \
104        html/context.html \
105        html/simple_all_pass_skip.html \
106        html/simple_some_fail_fail.html
107    do
108        test -f "${f}" || atf_fail "Missing ${f}"
109    done
110
111    atf_check -o match:"2 TESTS FAILING" cat html/index.html
112
113    check_in_file html/simple_all_pass_skip.html \
114        "This is the stdout of skip" "This is the stderr of skip"
115    check_not_in_file html/simple_all_pass_skip.html \
116        "This is the stdout of pass" "This is the stderr of pass" \
117        "This is the stdout of fail" "This is the stderr of fail" \
118        "Test case did not write anything to"
119
120    check_in_file html/simple_some_fail_fail.html \
121        "This is the stdout of fail" "This is the stderr of fail"
122    check_not_in_file html/simple_some_fail_fail.html \
123        "This is the stdout of pass" "This is the stderr of pass" \
124        "This is the stdout of skip" "This is the stderr of skip" \
125        "Test case did not write anything to"
126
127    check_in_file html/metadata_one_property.html \
128        "description = Does nothing but has one metadata property"
129    check_not_in_file html/metadata_one_property.html \
130        "allowed_architectures = some-architecture"
131
132    check_in_file html/metadata_many_properties.html \
133        "allowed_architectures = some-architecture"
134    check_not_in_file html/metadata_many_properties.html \
135        "description = Does nothing but has one metadata property"
136}
137
138
139utils_test_case default_behavior__no_store
140default_behavior__no_store_body() {
141    echo 'kyua: E: No previous results file found for test suite' \
142        "$(utils_test_suite_id)." >experr
143    atf_check -s exit:2 -o empty -e file:experr kyua report-html
144}
145
146
147utils_test_case results_file__explicit
148results_file__explicit_body() {
149    run_tests "mock1" dbfile_name1
150    run_tests "mock2" dbfile_name2
151
152    atf_check -s exit:0 -o ignore -e empty kyua report-html \
153        --results-file="$(cat dbfile_name1)"
154    grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report"
155
156    rm -rf html
157    atf_check -s exit:0 -o ignore -e empty kyua report-html \
158        --results-file="$(cat dbfile_name2)"
159    grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report"
160}
161
162
163utils_test_case results_file__not_found
164results_file__not_found_body() {
165    atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
166        kyua report-html --results-file=foo
167}
168
169
170utils_test_case force__yes
171force__yes_body() {
172    run_tests "mock1" unused_dbfile_name
173
174    atf_check -s exit:0 -o ignore -e empty kyua report-html
175    test -f html/index.html || atf_fail "Expected file not created"
176    rm html/index.html
177    atf_check -s exit:0 -o ignore -e empty kyua report-html --force
178    test -f html/index.html || atf_fail "Expected file not created"
179}
180
181
182utils_test_case force__no
183force__no_body() {
184    run_tests "mock1" unused_dbfile_name
185
186    atf_check -s exit:0 -o ignore -e empty kyua report-html
187    test -f html/index.html || atf_fail "Expected file not created"
188    rm html/index.html
189
190cat >experr <<EOF
191kyua: E: Output directory 'html' already exists; maybe use --force?.
192EOF
193    atf_check -s exit:2 -o empty -e file:experr kyua report-html
194    test ! -f html/index.html || atf_fail "Not expected file created"
195}
196
197
198utils_test_case output__explicit
199output__explicit_body() {
200    run_tests "mock1" unused_dbfile_name
201
202    mkdir output
203    atf_check -s exit:0 -o ignore -e empty kyua report-html --output=output/foo
204    test ! -d html || atf_fail "Not expected directory created"
205    test -f output/foo/index.html || atf_fail "Expected file not created"
206}
207
208
209utils_test_case results_filter__ok
210results_filter__ok_body() {
211    run_tests "mock1" unused_dbfile_name
212
213    atf_check -s exit:0 -o ignore -e empty kyua report-html \
214        --results-filter=passed
215    for f in \
216        html/index.html \
217        html/context.html \
218        html/simple_all_pass_pass.html \
219        html/simple_some_fail_pass.html \
220        html/metadata_no_properties.html \
221        html/metadata_with_cleanup.html
222    do
223        test -f "${f}" || atf_fail "Missing ${f}"
224    done
225
226    atf_check -o match:"2 TESTS FAILING" cat html/index.html
227
228    check_in_file html/simple_all_pass_pass.html \
229        "This is the stdout of pass" "This is the stderr of pass"
230    check_not_in_file html/simple_all_pass_pass.html \
231        "This is the stdout of skip" "This is the stderr of skip" \
232        "This is the stdout of fail" "This is the stderr of fail" \
233        "Test case did not write anything to"
234
235    check_in_file html/simple_some_fail_pass.html \
236        "Test case did not write anything to stdout" \
237        "Test case did not write anything to stderr"
238    check_not_in_file html/simple_some_fail_pass.html \
239        "This is the stdout of pass" "This is the stderr of pass" \
240        "This is the stdout of skip" "This is the stderr of skip" \
241        "This is the stdout of fail" "This is the stderr of fail"
242}
243
244
245utils_test_case results_filter__invalid
246results_filter__invalid_body() {
247    echo "kyua: E: Unknown result type 'foo-bar'." >experr
248    atf_check -s exit:2 -o empty -e file:experr kyua report-html \
249        --results-filter=passed,foo-bar
250}
251
252
253atf_init_test_cases() {
254    atf_add_test_case default_behavior__ok
255    atf_add_test_case default_behavior__no_store
256
257    atf_add_test_case results_file__explicit
258    atf_add_test_case results_file__not_found
259
260    atf_add_test_case force__yes
261    atf_add_test_case force__no
262
263    atf_add_test_case output__explicit
264
265    atf_add_test_case results_filter__ok
266    atf_add_test_case results_filter__invalid
267}
268