1 // Copyright 2015 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/tap.hpp" 30 31 extern "C" { 32 #include <signal.h> 33 } 34 35 #include <atf-c++.hpp> 36 37 #include "engine/config.hpp" 38 #include "engine/scheduler.hpp" 39 #include "model/metadata.hpp" 40 #include "model/test_case.hpp" 41 #include "model/test_program.hpp" 42 #include "model/test_result.hpp" 43 #include "utils/config/tree.ipp" 44 #include "utils/datetime.hpp" 45 #include "utils/env.hpp" 46 #include "utils/format/containers.ipp" 47 #include "utils/format/macros.hpp" 48 #include "utils/fs/operations.hpp" 49 #include "utils/fs/path.hpp" 50 #include "utils/optional.ipp" 51 52 namespace config = utils::config; 53 namespace datetime = utils::datetime; 54 namespace fs = utils::fs; 55 namespace scheduler = engine::scheduler; 56 57 using utils::none; 58 59 60 namespace { 61 62 63 /// Copies the tap helper to the work directory, selecting a specific helper. 64 /// 65 /// \param tc Pointer to the calling test case, to obtain srcdir. 66 /// \param name Name of the new binary to create. Must match the name of a 67 /// valid helper, as the binary name is used to select it. 68 static void 69 copy_tap_helper(const atf::tests::tc* tc, const char* name) 70 { 71 const fs::path srcdir(tc->get_config_var("srcdir")); 72 atf::utils::copy_file((srcdir / "tap_helpers").str(), name); 73 } 74 75 76 /// Runs one tap test program and checks its result. 77 /// 78 /// \param tc Pointer to the calling test case, to obtain srcdir. 79 /// \param test_case_name Name of the "test case" to select from the helper 80 /// program. 81 /// \param exp_result The expected result. 82 /// \param metadata The test case metadata. 83 /// \param user_config User-provided configuration variables. 84 static void 85 run_one(const atf::tests::tc* tc, const char* test_case_name, 86 const model::test_result& exp_result, 87 const model::metadata& metadata = model::metadata_builder().build(), 88 const config::tree& user_config = engine::empty_config()) 89 { 90 copy_tap_helper(tc, test_case_name); 91 const model::test_program_ptr program = model::test_program_builder( 92 "tap", fs::path(test_case_name), fs::current_path(), "the-suite") 93 .add_test_case("main", metadata).build_ptr(); 94 95 scheduler::scheduler_handle handle = scheduler::setup(); 96 (void)handle.spawn_test(program, "main", user_config); 97 98 scheduler::result_handle_ptr result_handle = handle.wait_any(); 99 const scheduler::test_result_handle* test_result_handle = 100 dynamic_cast< const scheduler::test_result_handle* >( 101 result_handle.get()); 102 atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); 103 atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); 104 ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); 105 result_handle->cleanup(); 106 result_handle.reset(); 107 108 handle.cleanup(); 109 } 110 111 112 } // anonymous namespace 113 114 115 ATF_TEST_CASE_WITHOUT_HEAD(list); 116 ATF_TEST_CASE_BODY(list) 117 { 118 const model::test_program program = model::test_program_builder( 119 "tap", fs::path("non-existent"), fs::path("."), "unused-suite") 120 .build(); 121 122 scheduler::scheduler_handle handle = scheduler::setup(); 123 const model::test_cases_map test_cases = handle.list_tests( 124 &program, engine::empty_config()); 125 handle.cleanup(); 126 127 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 128 .add("main").build(); 129 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 130 } 131 132 133 ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass); 134 ATF_TEST_CASE_BODY(test__all_tests_pass) 135 { 136 const model::test_result exp_result(model::test_result_passed); 137 run_one(this, "pass", exp_result); 138 } 139 140 141 ATF_TEST_CASE_WITHOUT_HEAD(test__some_tests_fail); 142 ATF_TEST_CASE_BODY(test__some_tests_fail) 143 { 144 const model::test_result exp_result(model::test_result_failed, 145 "2 of 5 tests failed"); 146 run_one(this, "fail", exp_result); 147 } 148 149 150 ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass_but_exit_failure); 151 ATF_TEST_CASE_BODY(test__all_tests_pass_but_exit_failure) 152 { 153 const model::test_result exp_result( 154 model::test_result_broken, 155 "Dubious test program: reported all tests as passed but returned exit " 156 "code 70"); 157 run_one(this, "pass_but_exit_failure", exp_result); 158 } 159 160 161 ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken); 162 ATF_TEST_CASE_BODY(test__signal_is_broken) 163 { 164 const model::test_result exp_result(model::test_result_broken, 165 F("Received signal %s") % SIGABRT); 166 run_one(this, "crash", exp_result); 167 } 168 169 170 ATF_TEST_CASE(test__timeout_is_broken); 171 ATF_TEST_CASE_HEAD(test__timeout_is_broken) 172 { 173 set_md_var("timeout", "60"); 174 } 175 ATF_TEST_CASE_BODY(test__timeout_is_broken) 176 { 177 utils::setenv("CONTROL_DIR", fs::current_path().str()); 178 179 const model::metadata metadata = model::metadata_builder() 180 .set_timeout(datetime::delta(1, 0)).build(); 181 const model::test_result exp_result(model::test_result_broken, 182 "Test case timed out"); 183 run_one(this, "timeout", exp_result, metadata); 184 185 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 186 } 187 188 189 ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables); 190 ATF_TEST_CASE_BODY(test__configuration_variables) 191 { 192 config::tree user_config = engine::empty_config(); 193 user_config.set_string("test_suites.a-suite.first", "unused"); 194 user_config.set_string("test_suites.the-suite.first", "some value"); 195 user_config.set_string("test_suites.the-suite.second", "some other value"); 196 user_config.set_string("test_suites.other-suite.first", "unused"); 197 198 const model::test_result exp_result(model::test_result_passed); 199 run_one(this, "check_configuration_variables", exp_result, 200 model::metadata_builder().build(), user_config); 201 } 202 203 204 ATF_INIT_TEST_CASES(tcs) 205 { 206 scheduler::register_interface( 207 "tap", std::shared_ptr< scheduler::interface >( 208 new engine::tap_interface())); 209 210 ATF_ADD_TEST_CASE(tcs, list); 211 212 ATF_ADD_TEST_CASE(tcs, test__all_tests_pass); 213 ATF_ADD_TEST_CASE(tcs, test__all_tests_pass_but_exit_failure); 214 ATF_ADD_TEST_CASE(tcs, test__some_tests_fail); 215 ATF_ADD_TEST_CASE(tcs, test__signal_is_broken); 216 ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken); 217 ATF_ADD_TEST_CASE(tcs, test__configuration_variables); 218 } 219