1 // Copyright 2011 Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/drivers/run_tests.hpp" 30 31 #include "engine/action.hpp" 32 #include "engine/context.hpp" 33 #include "engine/filters.hpp" 34 #include "engine/kyuafile.hpp" 35 #include "engine/test_program.hpp" 36 #include "engine/test_result.hpp" 37 #include "store/backend.hpp" 38 #include "store/transaction.hpp" 39 #include "utils/datetime.hpp" 40 #include "utils/defs.hpp" 41 #include "utils/format/macros.hpp" 42 #include "utils/fs/auto_cleaners.hpp" 43 #include "utils/logging/macros.hpp" 44 #include "utils/optional.ipp" 45 #include "utils/signals/interrupts.hpp" 46 47 namespace config = utils::config; 48 namespace datetime = utils::datetime; 49 namespace fs = utils::fs; 50 namespace run_tests = engine::drivers::run_tests; 51 namespace signals = utils::signals; 52 53 using utils::optional; 54 55 56 namespace { 57 58 59 /// Test case hooks to save the output into the database. 60 class file_saver_hooks : public engine::test_case_hooks { 61 /// Open write transaction for the test case's data. 62 store::transaction& _tx; 63 64 /// Identifier of the test case being stored. 65 const int64_t _test_case_id; 66 67 public: 68 /// Constructs a new set of hooks. 69 /// 70 /// \param tx_ Open write transaction for the test case's data. 71 /// \param test_case_id_ Identifier of the test case being stored. 72 file_saver_hooks(store::transaction& tx_, 73 const int64_t test_case_id_) : 74 _tx(tx_), _test_case_id(test_case_id_) 75 { 76 } 77 78 /// Stores the stdout of the test case into the database. 79 /// 80 /// \param file Path to the stdout of the test case. 81 void 82 got_stdout(const fs::path& file) 83 { 84 _tx.put_test_case_file("__STDOUT__", file, _test_case_id); 85 } 86 87 /// Stores the stderr of the test case into the database. 88 /// 89 /// \param file Path to the stderr of the test case. 90 void 91 got_stderr(const fs::path& file) 92 { 93 _tx.put_test_case_file("__STDERR__", file, _test_case_id); 94 } 95 }; 96 97 98 /// Runs a test program in a controlled manner. 99 /// 100 /// If the test program fails to provide a list of test cases, a fake test case 101 /// named '__test_program__' is created and it is reported as broken. 102 /// 103 /// \param program The test program to execute. 104 /// \param user_config The configuration variables provided by the user. 105 /// \param filters The matching state of the filters. 106 /// \param hooks The user hooks to receive asynchronous notifications. 107 /// \param work_directory Temporary directory to use. 108 /// \param tx The store transaction into which to put the results. 109 /// \param action_id The action this program belongs to. 110 void 111 run_test_program(const engine::test_program& program, 112 const config::tree& user_config, 113 engine::filters_state& filters, 114 run_tests::base_hooks& hooks, 115 const fs::path& work_directory, 116 store::transaction& tx, 117 const int64_t action_id) 118 { 119 LI(F("Processing test program '%s'") % program.relative_path()); 120 const int64_t test_program_id = tx.put_test_program(program, action_id); 121 122 const engine::test_cases_vector& test_cases = program.test_cases(); 123 for (engine::test_cases_vector::const_iterator iter = test_cases.begin(); 124 iter != test_cases.end(); iter++) { 125 const engine::test_case_ptr test_case = *iter; 126 127 if (!filters.match_test_case(program.relative_path(), 128 test_case->name())) 129 continue; 130 131 const int64_t test_case_id = tx.put_test_case(*test_case, 132 test_program_id); 133 file_saver_hooks test_hooks(tx, test_case_id); 134 hooks.got_test_case(test_case); 135 const datetime::timestamp start_time = datetime::timestamp::now(); 136 const engine::test_result result = run_test_case( 137 test_case.get(), user_config, test_hooks, work_directory); 138 const datetime::timestamp end_time = datetime::timestamp::now(); 139 tx.put_result(result, test_case_id, start_time, end_time); 140 hooks.got_result(test_case, result, end_time - start_time); 141 142 signals::check_interrupt(); 143 } 144 } 145 146 147 } // anonymous namespace 148 149 150 /// Pure abstract destructor. 151 run_tests::base_hooks::~base_hooks(void) 152 { 153 } 154 155 156 /// Executes the operation. 157 /// 158 /// \param kyuafile_path The path to the Kyuafile to be loaded. 159 /// \param build_root If not none, path to the built test programs. 160 /// \param store_path The path to the store to be used. 161 /// \param raw_filters The test case filters as provided by the user. 162 /// \param user_config The end-user configuration properties. 163 /// \param hooks The hooks for this execution. 164 /// 165 /// \returns A structure with all results computed by this driver. 166 run_tests::result 167 run_tests::drive(const fs::path& kyuafile_path, 168 const optional< fs::path > build_root, 169 const fs::path& store_path, 170 const std::set< engine::test_filter >& raw_filters, 171 const config::tree& user_config, 172 base_hooks& hooks) 173 { 174 const engine::kyuafile kyuafile = engine::kyuafile::load( 175 kyuafile_path, build_root); 176 filters_state filters(raw_filters); 177 store::backend db = store::backend::open_rw(store_path); 178 store::transaction tx = db.start(); 179 180 engine::context context = engine::context::current(); 181 const int64_t context_id = tx.put_context(context); 182 183 engine::action action(context); 184 const int64_t action_id = tx.put_action(action, context_id); 185 186 signals::interrupts_handler interrupts; 187 188 const fs::auto_directory work_directory = fs::auto_directory::mkdtemp( 189 "kyua.XXXXXX"); 190 191 for (test_programs_vector::const_iterator iter = 192 kyuafile.test_programs().begin(); 193 iter != kyuafile.test_programs().end(); iter++) { 194 const test_program_ptr& test_program = *iter; 195 196 if (!filters.match_test_program(test_program->relative_path())) 197 continue; 198 199 run_test_program(*test_program, user_config, filters, hooks, 200 work_directory.directory(), tx, action_id); 201 202 signals::check_interrupt(); 203 } 204 205 tx.commit(); 206 207 return result(action_id, filters.unused()); 208 } 209