1*b0d29bc4SBrooks Davis // Copyright 2015 The Kyua Authors.
2*b0d29bc4SBrooks Davis // All rights reserved.
3*b0d29bc4SBrooks Davis //
4*b0d29bc4SBrooks Davis // Redistribution and use in source and binary forms, with or without
5*b0d29bc4SBrooks Davis // modification, are permitted provided that the following conditions are
6*b0d29bc4SBrooks Davis // met:
7*b0d29bc4SBrooks Davis //
8*b0d29bc4SBrooks Davis // * Redistributions of source code must retain the above copyright
9*b0d29bc4SBrooks Davis // notice, this list of conditions and the following disclaimer.
10*b0d29bc4SBrooks Davis // * Redistributions in binary form must reproduce the above copyright
11*b0d29bc4SBrooks Davis // notice, this list of conditions and the following disclaimer in the
12*b0d29bc4SBrooks Davis // documentation and/or other materials provided with the distribution.
13*b0d29bc4SBrooks Davis // * Neither the name of Google Inc. nor the names of its contributors
14*b0d29bc4SBrooks Davis // may be used to endorse or promote products derived from this software
15*b0d29bc4SBrooks Davis // without specific prior written permission.
16*b0d29bc4SBrooks Davis //
17*b0d29bc4SBrooks Davis // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18*b0d29bc4SBrooks Davis // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19*b0d29bc4SBrooks Davis // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20*b0d29bc4SBrooks Davis // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21*b0d29bc4SBrooks Davis // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22*b0d29bc4SBrooks Davis // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23*b0d29bc4SBrooks Davis // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24*b0d29bc4SBrooks Davis // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25*b0d29bc4SBrooks Davis // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26*b0d29bc4SBrooks Davis // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27*b0d29bc4SBrooks Davis // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28*b0d29bc4SBrooks Davis
29*b0d29bc4SBrooks Davis #include "engine/tap.hpp"
30*b0d29bc4SBrooks Davis
31*b0d29bc4SBrooks Davis extern "C" {
32*b0d29bc4SBrooks Davis #include <signal.h>
33*b0d29bc4SBrooks Davis }
34*b0d29bc4SBrooks Davis
35*b0d29bc4SBrooks Davis #include <atf-c++.hpp>
36*b0d29bc4SBrooks Davis
37*b0d29bc4SBrooks Davis #include "engine/config.hpp"
38*b0d29bc4SBrooks Davis #include "engine/scheduler.hpp"
39*b0d29bc4SBrooks Davis #include "model/metadata.hpp"
40*b0d29bc4SBrooks Davis #include "model/test_case.hpp"
41*b0d29bc4SBrooks Davis #include "model/test_program.hpp"
42*b0d29bc4SBrooks Davis #include "model/test_result.hpp"
43*b0d29bc4SBrooks Davis #include "utils/config/tree.ipp"
44*b0d29bc4SBrooks Davis #include "utils/datetime.hpp"
45*b0d29bc4SBrooks Davis #include "utils/env.hpp"
46*b0d29bc4SBrooks Davis #include "utils/format/containers.ipp"
47*b0d29bc4SBrooks Davis #include "utils/format/macros.hpp"
48*b0d29bc4SBrooks Davis #include "utils/fs/operations.hpp"
49*b0d29bc4SBrooks Davis #include "utils/fs/path.hpp"
50*b0d29bc4SBrooks Davis #include "utils/optional.ipp"
51*b0d29bc4SBrooks Davis
52*b0d29bc4SBrooks Davis namespace config = utils::config;
53*b0d29bc4SBrooks Davis namespace datetime = utils::datetime;
54*b0d29bc4SBrooks Davis namespace fs = utils::fs;
55*b0d29bc4SBrooks Davis namespace scheduler = engine::scheduler;
56*b0d29bc4SBrooks Davis
57*b0d29bc4SBrooks Davis using utils::none;
58*b0d29bc4SBrooks Davis
59*b0d29bc4SBrooks Davis
60*b0d29bc4SBrooks Davis namespace {
61*b0d29bc4SBrooks Davis
62*b0d29bc4SBrooks Davis
63*b0d29bc4SBrooks Davis /// Copies the tap helper to the work directory, selecting a specific helper.
64*b0d29bc4SBrooks Davis ///
65*b0d29bc4SBrooks Davis /// \param tc Pointer to the calling test case, to obtain srcdir.
66*b0d29bc4SBrooks Davis /// \param name Name of the new binary to create. Must match the name of a
67*b0d29bc4SBrooks Davis /// valid helper, as the binary name is used to select it.
68*b0d29bc4SBrooks Davis static void
copy_tap_helper(const atf::tests::tc * tc,const char * name)69*b0d29bc4SBrooks Davis copy_tap_helper(const atf::tests::tc* tc, const char* name)
70*b0d29bc4SBrooks Davis {
71*b0d29bc4SBrooks Davis const fs::path srcdir(tc->get_config_var("srcdir"));
72*b0d29bc4SBrooks Davis atf::utils::copy_file((srcdir / "tap_helpers").str(), name);
73*b0d29bc4SBrooks Davis }
74*b0d29bc4SBrooks Davis
75*b0d29bc4SBrooks Davis
76*b0d29bc4SBrooks Davis /// Runs one tap test program and checks its result.
77*b0d29bc4SBrooks Davis ///
78*b0d29bc4SBrooks Davis /// \param tc Pointer to the calling test case, to obtain srcdir.
79*b0d29bc4SBrooks Davis /// \param test_case_name Name of the "test case" to select from the helper
80*b0d29bc4SBrooks Davis /// program.
81*b0d29bc4SBrooks Davis /// \param exp_result The expected result.
82*b0d29bc4SBrooks Davis /// \param metadata The test case metadata.
83*b0d29bc4SBrooks Davis /// \param user_config User-provided configuration variables.
84*b0d29bc4SBrooks Davis static void
run_one(const atf::tests::tc * tc,const char * test_case_name,const model::test_result & exp_result,const model::metadata & metadata=model::metadata_builder ().build (),const config::tree & user_config=engine::empty_config ())85*b0d29bc4SBrooks Davis run_one(const atf::tests::tc* tc, const char* test_case_name,
86*b0d29bc4SBrooks Davis const model::test_result& exp_result,
87*b0d29bc4SBrooks Davis const model::metadata& metadata = model::metadata_builder().build(),
88*b0d29bc4SBrooks Davis const config::tree& user_config = engine::empty_config())
89*b0d29bc4SBrooks Davis {
90*b0d29bc4SBrooks Davis copy_tap_helper(tc, test_case_name);
91*b0d29bc4SBrooks Davis const model::test_program_ptr program = model::test_program_builder(
92*b0d29bc4SBrooks Davis "tap", fs::path(test_case_name), fs::current_path(), "the-suite")
93*b0d29bc4SBrooks Davis .add_test_case("main", metadata).build_ptr();
94*b0d29bc4SBrooks Davis
95*b0d29bc4SBrooks Davis scheduler::scheduler_handle handle = scheduler::setup();
96*b0d29bc4SBrooks Davis (void)handle.spawn_test(program, "main", user_config);
97*b0d29bc4SBrooks Davis
98*b0d29bc4SBrooks Davis scheduler::result_handle_ptr result_handle = handle.wait_any();
99*b0d29bc4SBrooks Davis const scheduler::test_result_handle* test_result_handle =
100*b0d29bc4SBrooks Davis dynamic_cast< const scheduler::test_result_handle* >(
101*b0d29bc4SBrooks Davis result_handle.get());
102*b0d29bc4SBrooks Davis atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: ");
103*b0d29bc4SBrooks Davis atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: ");
104*b0d29bc4SBrooks Davis ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
105*b0d29bc4SBrooks Davis result_handle->cleanup();
106*b0d29bc4SBrooks Davis result_handle.reset();
107*b0d29bc4SBrooks Davis
108*b0d29bc4SBrooks Davis handle.cleanup();
109*b0d29bc4SBrooks Davis }
110*b0d29bc4SBrooks Davis
111*b0d29bc4SBrooks Davis
112*b0d29bc4SBrooks Davis } // anonymous namespace
113*b0d29bc4SBrooks Davis
114*b0d29bc4SBrooks Davis
115*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(list);
ATF_TEST_CASE_BODY(list)116*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(list)
117*b0d29bc4SBrooks Davis {
118*b0d29bc4SBrooks Davis const model::test_program program = model::test_program_builder(
119*b0d29bc4SBrooks Davis "tap", fs::path("non-existent"), fs::path("."), "unused-suite")
120*b0d29bc4SBrooks Davis .build();
121*b0d29bc4SBrooks Davis
122*b0d29bc4SBrooks Davis scheduler::scheduler_handle handle = scheduler::setup();
123*b0d29bc4SBrooks Davis const model::test_cases_map test_cases = handle.list_tests(
124*b0d29bc4SBrooks Davis &program, engine::empty_config());
125*b0d29bc4SBrooks Davis handle.cleanup();
126*b0d29bc4SBrooks Davis
127*b0d29bc4SBrooks Davis const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
128*b0d29bc4SBrooks Davis .add("main").build();
129*b0d29bc4SBrooks Davis ATF_REQUIRE_EQ(exp_test_cases, test_cases);
130*b0d29bc4SBrooks Davis }
131*b0d29bc4SBrooks Davis
132*b0d29bc4SBrooks Davis
133*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass);
ATF_TEST_CASE_BODY(test__all_tests_pass)134*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__all_tests_pass)
135*b0d29bc4SBrooks Davis {
136*b0d29bc4SBrooks Davis const model::test_result exp_result(model::test_result_passed);
137*b0d29bc4SBrooks Davis run_one(this, "pass", exp_result);
138*b0d29bc4SBrooks Davis }
139*b0d29bc4SBrooks Davis
140*b0d29bc4SBrooks Davis
141*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(test__some_tests_fail);
ATF_TEST_CASE_BODY(test__some_tests_fail)142*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__some_tests_fail)
143*b0d29bc4SBrooks Davis {
144*b0d29bc4SBrooks Davis const model::test_result exp_result(model::test_result_failed,
145*b0d29bc4SBrooks Davis "2 of 5 tests failed");
146*b0d29bc4SBrooks Davis run_one(this, "fail", exp_result);
147*b0d29bc4SBrooks Davis }
148*b0d29bc4SBrooks Davis
149*b0d29bc4SBrooks Davis
150*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass_but_exit_failure);
ATF_TEST_CASE_BODY(test__all_tests_pass_but_exit_failure)151*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__all_tests_pass_but_exit_failure)
152*b0d29bc4SBrooks Davis {
153*b0d29bc4SBrooks Davis const model::test_result exp_result(
154*b0d29bc4SBrooks Davis model::test_result_broken,
155*b0d29bc4SBrooks Davis "Dubious test program: reported all tests as passed but returned exit "
156*b0d29bc4SBrooks Davis "code 70");
157*b0d29bc4SBrooks Davis run_one(this, "pass_but_exit_failure", exp_result);
158*b0d29bc4SBrooks Davis }
159*b0d29bc4SBrooks Davis
160*b0d29bc4SBrooks Davis
161*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken);
ATF_TEST_CASE_BODY(test__signal_is_broken)162*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__signal_is_broken)
163*b0d29bc4SBrooks Davis {
164*b0d29bc4SBrooks Davis const model::test_result exp_result(model::test_result_broken,
165*b0d29bc4SBrooks Davis F("Received signal %s") % SIGABRT);
166*b0d29bc4SBrooks Davis run_one(this, "crash", exp_result);
167*b0d29bc4SBrooks Davis }
168*b0d29bc4SBrooks Davis
169*b0d29bc4SBrooks Davis
170*b0d29bc4SBrooks Davis ATF_TEST_CASE(test__timeout_is_broken);
ATF_TEST_CASE_HEAD(test__timeout_is_broken)171*b0d29bc4SBrooks Davis ATF_TEST_CASE_HEAD(test__timeout_is_broken)
172*b0d29bc4SBrooks Davis {
173*b0d29bc4SBrooks Davis set_md_var("timeout", "60");
174*b0d29bc4SBrooks Davis }
ATF_TEST_CASE_BODY(test__timeout_is_broken)175*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__timeout_is_broken)
176*b0d29bc4SBrooks Davis {
177*b0d29bc4SBrooks Davis utils::setenv("CONTROL_DIR", fs::current_path().str());
178*b0d29bc4SBrooks Davis
179*b0d29bc4SBrooks Davis const model::metadata metadata = model::metadata_builder()
180*b0d29bc4SBrooks Davis .set_timeout(datetime::delta(1, 0)).build();
181*b0d29bc4SBrooks Davis const model::test_result exp_result(model::test_result_broken,
182*b0d29bc4SBrooks Davis "Test case timed out");
183*b0d29bc4SBrooks Davis run_one(this, "timeout", exp_result, metadata);
184*b0d29bc4SBrooks Davis
185*b0d29bc4SBrooks Davis ATF_REQUIRE(!atf::utils::file_exists("cookie"));
186*b0d29bc4SBrooks Davis }
187*b0d29bc4SBrooks Davis
188*b0d29bc4SBrooks Davis
189*b0d29bc4SBrooks Davis ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables);
ATF_TEST_CASE_BODY(test__configuration_variables)190*b0d29bc4SBrooks Davis ATF_TEST_CASE_BODY(test__configuration_variables)
191*b0d29bc4SBrooks Davis {
192*b0d29bc4SBrooks Davis config::tree user_config = engine::empty_config();
193*b0d29bc4SBrooks Davis user_config.set_string("test_suites.a-suite.first", "unused");
194*b0d29bc4SBrooks Davis user_config.set_string("test_suites.the-suite.first", "some value");
195*b0d29bc4SBrooks Davis user_config.set_string("test_suites.the-suite.second", "some other value");
196*b0d29bc4SBrooks Davis user_config.set_string("test_suites.other-suite.first", "unused");
197*b0d29bc4SBrooks Davis
198*b0d29bc4SBrooks Davis const model::test_result exp_result(model::test_result_passed);
199*b0d29bc4SBrooks Davis run_one(this, "check_configuration_variables", exp_result,
200*b0d29bc4SBrooks Davis model::metadata_builder().build(), user_config);
201*b0d29bc4SBrooks Davis }
202*b0d29bc4SBrooks Davis
203*b0d29bc4SBrooks Davis
ATF_INIT_TEST_CASES(tcs)204*b0d29bc4SBrooks Davis ATF_INIT_TEST_CASES(tcs)
205*b0d29bc4SBrooks Davis {
206*b0d29bc4SBrooks Davis scheduler::register_interface(
207*b0d29bc4SBrooks Davis "tap", std::shared_ptr< scheduler::interface >(
208*b0d29bc4SBrooks Davis new engine::tap_interface()));
209*b0d29bc4SBrooks Davis
210*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, list);
211*b0d29bc4SBrooks Davis
212*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__all_tests_pass);
213*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__all_tests_pass_but_exit_failure);
214*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__some_tests_fail);
215*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__signal_is_broken);
216*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken);
217*b0d29bc4SBrooks Davis ATF_ADD_TEST_CASE(tcs, test__configuration_variables);
218*b0d29bc4SBrooks Davis }
219