1 // Copyright 2010 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 //   notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 //   notice, this list of conditions and the following disclaimer in the
12 //   documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 //   may be used to endorse or promote products derived from this software
15 //   without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include "engine/test_program.hpp"
30 
31 extern "C" {
32 #include <sys/stat.h>
33 
34 #include <signal.h>
35 }
36 
37 #include <sstream>
38 
39 #include <atf-c++.hpp>
40 
41 #include "engine/exceptions.hpp"
42 #include "engine/test_result.hpp"
43 #include "utils/env.hpp"
44 #include "utils/format/macros.hpp"
45 #include "utils/fs/operations.hpp"
46 #include "utils/fs/path.hpp"
47 #include "utils/optional.ipp"
48 
49 namespace fs = utils::fs;
50 
51 
52 namespace {
53 
54 
55 /// Creates a mock tester that receives a signal.
56 ///
57 /// \param term_sig Signal to deliver to the tester.  If the tester does not
58 ///     exit due to this reason, it exits with an arbitrary non-zero code.
59 static void
create_mock_tester_signal(const int term_sig)60 create_mock_tester_signal(const int term_sig)
61 {
62     const std::string tester_name = "kyua-mock-tester";
63 
64     atf::utils::create_file(
65         tester_name,
66         F("#! /bin/sh\n"
67           "kill -%s $$\n"
68           "exit 0\n") % term_sig);
69     ATF_REQUIRE(::chmod(tester_name.c_str(), 0755) != -1);
70 
71     utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
72 }
73 
74 
75 }  // anonymous namespace
76 
77 
78 ATF_TEST_CASE_WITHOUT_HEAD(ctor_and_getters);
ATF_TEST_CASE_BODY(ctor_and_getters)79 ATF_TEST_CASE_BODY(ctor_and_getters)
80 {
81     const engine::metadata md = engine::metadata_builder()
82         .add_custom("foo", "bar")
83         .build();
84     const engine::test_program test_program(
85         "mock", fs::path("binary"), fs::path("root"), "suite-name", md);
86     ATF_REQUIRE_EQ("mock", test_program.interface_name());
87     ATF_REQUIRE_EQ(fs::path("binary"), test_program.relative_path());
88     ATF_REQUIRE_EQ(fs::current_path() / "root/binary",
89                    test_program.absolute_path());
90     ATF_REQUIRE_EQ(fs::path("root"), test_program.root());
91     ATF_REQUIRE_EQ("suite-name", test_program.test_suite_name());
92     ATF_REQUIRE_EQ(md, test_program.get_metadata());
93 }
94 
95 
96 ATF_TEST_CASE_WITHOUT_HEAD(find__ok);
ATF_TEST_CASE_BODY(find__ok)97 ATF_TEST_CASE_BODY(find__ok)
98 {
99     const engine::test_program test_program(
100         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
101         engine::metadata_builder().build());
102     const engine::test_case_ptr test_case = test_program.find("main");
103     ATF_REQUIRE_EQ(fs::path("non-existent"),
104                    test_case->container_test_program().relative_path());
105     ATF_REQUIRE_EQ("main", test_case->name());
106 }
107 
108 
109 ATF_TEST_CASE_WITHOUT_HEAD(find__missing);
ATF_TEST_CASE_BODY(find__missing)110 ATF_TEST_CASE_BODY(find__missing)
111 {
112     const engine::test_program test_program(
113         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
114         engine::metadata_builder().build());
115     ATF_REQUIRE_THROW_RE(engine::not_found_error,
116                          "case.*abc.*program.*non-existent",
117                          test_program.find("abc"));
118 }
119 
120 
121 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__get);
ATF_TEST_CASE_BODY(test_cases__get)122 ATF_TEST_CASE_BODY(test_cases__get)
123 {
124     const engine::test_program test_program(
125         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
126         engine::metadata_builder().build());
127     const engine::test_cases_vector& test_cases = test_program.test_cases();
128     ATF_REQUIRE_EQ(1, test_cases.size());
129     ATF_REQUIRE_EQ(fs::path("non-existent"),
130                    test_cases[0]->container_test_program().relative_path());
131     ATF_REQUIRE_EQ("main", test_cases[0]->name());
132 }
133 
134 
135 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__some);
ATF_TEST_CASE_BODY(test_cases__some)136 ATF_TEST_CASE_BODY(test_cases__some)
137 {
138     engine::test_program test_program(
139         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
140         engine::metadata_builder().build());
141 
142     engine::test_cases_vector exp_test_cases;
143     const engine::test_case test_case("plain", test_program, "main",
144                                       engine::metadata_builder().build());
145     exp_test_cases.push_back(engine::test_case_ptr(
146         new engine::test_case(test_case)));
147     test_program.set_test_cases(exp_test_cases);
148 
149     ATF_REQUIRE_EQ(exp_test_cases, test_program.test_cases());
150 }
151 
152 
153 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__tester_fails);
ATF_TEST_CASE_BODY(test_cases__tester_fails)154 ATF_TEST_CASE_BODY(test_cases__tester_fails)
155 {
156     engine::test_program test_program(
157         "mock", fs::path("non-existent"), fs::path("."), "suite-name",
158         engine::metadata_builder().build());
159     create_mock_tester_signal(SIGSEGV);
160 
161     const engine::test_cases_vector& test_cases = test_program.test_cases();
162     ATF_REQUIRE_EQ(1, test_cases.size());
163 
164     const engine::test_case_ptr& test_case = test_cases[0];
165     ATF_REQUIRE_EQ("__test_cases_list__", test_case->name());
166 
167     ATF_REQUIRE(test_case->fake_result());
168     const engine::test_result result = test_case->fake_result().get();
169     ATF_REQUIRE(engine::test_result::broken == result.type());
170     ATF_REQUIRE_MATCH("Tester did not exit cleanly", result.reason());
171 }
172 
173 
174 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__copy);
ATF_TEST_CASE_BODY(operators_eq_and_ne__copy)175 ATF_TEST_CASE_BODY(operators_eq_and_ne__copy)
176 {
177     const engine::test_program tp1(
178         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
179         engine::metadata_builder().build());
180     const engine::test_program tp2 = tp1;
181     ATF_REQUIRE(  tp1 == tp2);
182     ATF_REQUIRE(!(tp1 != tp2));
183 }
184 
185 
186 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__not_copy);
ATF_TEST_CASE_BODY(operators_eq_and_ne__not_copy)187 ATF_TEST_CASE_BODY(operators_eq_and_ne__not_copy)
188 {
189     const std::string base_interface("plain");
190     const fs::path base_relative_path("the/test/program");
191     const fs::path base_root("/the/root");
192     const std::string base_test_suite("suite-name");
193     const engine::metadata base_metadata = engine::metadata_builder()
194         .add_custom("X-foo", "bar")
195         .build();
196 
197     engine::test_program base_tp(
198         base_interface, base_relative_path, base_root, base_test_suite,
199         base_metadata);
200 
201     engine::test_cases_vector base_tcs;
202     {
203         const engine::test_case tc1("plain", base_tp, "main",
204                                     engine::metadata_builder().build());
205         base_tcs.push_back(engine::test_case_ptr(new engine::test_case(tc1)));
206     }
207     base_tp.set_test_cases(base_tcs);
208 
209     // Construct with all same values.
210     {
211         engine::test_program other_tp(
212             base_interface, base_relative_path, base_root, base_test_suite,
213             base_metadata);
214 
215         engine::test_cases_vector other_tcs;
216         {
217             const engine::test_case tc1("plain", other_tp, "main",
218                                         engine::metadata_builder().build());
219             other_tcs.push_back(engine::test_case_ptr(
220                 new engine::test_case(tc1)));
221         }
222         other_tp.set_test_cases(other_tcs);
223 
224         ATF_REQUIRE(  base_tp == other_tp);
225         ATF_REQUIRE(!(base_tp != other_tp));
226     }
227 
228     // Different interface.
229     {
230         engine::test_program other_tp(
231             "atf", base_relative_path, base_root, base_test_suite,
232             base_metadata);
233         other_tp.set_test_cases(base_tcs);
234 
235         ATF_REQUIRE(!(base_tp == other_tp));
236         ATF_REQUIRE(  base_tp != other_tp);
237     }
238 
239     // Different relative path.
240     {
241         engine::test_program other_tp(
242             base_interface, fs::path("a/b/c"), base_root, base_test_suite,
243             base_metadata);
244         other_tp.set_test_cases(base_tcs);
245 
246         ATF_REQUIRE(!(base_tp == other_tp));
247         ATF_REQUIRE(  base_tp != other_tp);
248     }
249 
250     // Different root.
251     {
252         engine::test_program other_tp(
253             base_interface, base_relative_path, fs::path("."), base_test_suite,
254             base_metadata);
255         other_tp.set_test_cases(base_tcs);
256 
257         ATF_REQUIRE(!(base_tp == other_tp));
258         ATF_REQUIRE(  base_tp != other_tp);
259     }
260 
261     // Different test suite.
262     {
263         engine::test_program other_tp(
264             base_interface, base_relative_path, base_root, "different-suite",
265             base_metadata);
266         other_tp.set_test_cases(base_tcs);
267 
268         ATF_REQUIRE(!(base_tp == other_tp));
269         ATF_REQUIRE(  base_tp != other_tp);
270     }
271 
272     // Different metadata.
273     {
274         engine::test_program other_tp(
275             base_interface, base_relative_path, base_root, base_test_suite,
276             engine::metadata_builder().build());
277         other_tp.set_test_cases(base_tcs);
278 
279         ATF_REQUIRE(!(base_tp == other_tp));
280         ATF_REQUIRE(  base_tp != other_tp);
281     }
282 
283     // Different test cases.
284     {
285         engine::test_program other_tp(
286             base_interface, base_relative_path, base_root, base_test_suite,
287             base_metadata);
288 
289         engine::test_cases_vector other_tcs;
290         {
291             const engine::test_case tc1("atf", base_tp, "foo",
292                                         engine::metadata_builder().build());
293             other_tcs.push_back(engine::test_case_ptr(
294                                     new engine::test_case(tc1)));
295         }
296         other_tp.set_test_cases(other_tcs);
297 
298         ATF_REQUIRE(!(base_tp == other_tp));
299         ATF_REQUIRE(  base_tp != other_tp);
300     }
301 }
302 
303 
304 ATF_TEST_CASE_WITHOUT_HEAD(output__no_test_cases);
ATF_TEST_CASE_BODY(output__no_test_cases)305 ATF_TEST_CASE_BODY(output__no_test_cases)
306 {
307     engine::test_program tp(
308         "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
309         engine::metadata_builder().add_allowed_architecture("a").build());
310     tp.set_test_cases(engine::test_cases_vector());
311 
312     std::ostringstream str;
313     str << tp;
314     ATF_REQUIRE_EQ(
315         "test_program{interface='plain', binary='binary/path', "
316         "root='/the/root', test_suite='suite-name', "
317         "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
318         "description='', has_cleanup='false', "
319         "required_configs='', required_files='', required_memory='0', "
320         "required_programs='', required_user='', timeout='300'}, "
321         "test_cases=[]}",
322         str.str());
323 }
324 
325 
326 ATF_TEST_CASE_WITHOUT_HEAD(output__some_test_cases);
ATF_TEST_CASE_BODY(output__some_test_cases)327 ATF_TEST_CASE_BODY(output__some_test_cases)
328 {
329     engine::test_program tp(
330         "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
331         engine::metadata_builder().add_allowed_architecture("a").build());
332 
333     const engine::test_case_ptr tc1(new engine::test_case(
334         "plain", tp, "the-name", engine::metadata_builder()
335         .add_allowed_platform("foo").add_custom("X-bar", "baz").build()));
336     const engine::test_case_ptr tc2(new engine::test_case(
337         "plain", tp, "another-name", engine::metadata_builder().build()));
338     engine::test_cases_vector tcs;
339     tcs.push_back(tc1);
340     tcs.push_back(tc2);
341     tp.set_test_cases(tcs);
342 
343     std::ostringstream str;
344     str << tp;
345     ATF_REQUIRE_EQ(
346         "test_program{interface='plain', binary='binary/path', "
347         "root='/the/root', test_suite='suite-name', "
348         "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
349         "description='', has_cleanup='false', "
350         "required_configs='', required_files='', required_memory='0', "
351         "required_programs='', required_user='', timeout='300'}, "
352         "test_cases=["
353         "test_case{interface='plain', name='the-name', "
354         "metadata=metadata{allowed_architectures='', allowed_platforms='foo', "
355         "custom.X-bar='baz', description='', has_cleanup='false', "
356         "required_configs='', required_files='', required_memory='0', "
357         "required_programs='', required_user='', timeout='300'}}, "
358         "test_case{interface='plain', name='another-name', "
359         "metadata=metadata{allowed_architectures='', allowed_platforms='', "
360         "description='', has_cleanup='false', "
361         "required_configs='', required_files='', required_memory='0', "
362         "required_programs='', required_user='', timeout='300'}}]}",
363         str.str());
364 }
365 
366 
ATF_INIT_TEST_CASES(tcs)367 ATF_INIT_TEST_CASES(tcs)
368 {
369     // TODO(jmmv): These tests have ceased to be realistic with the move to
370     // TestersDesign.  We probably should have some (few!) integration tests for
371     // the various known testers... or, alternatively, provide a mock tester to
372     // run our tests with.
373     ATF_ADD_TEST_CASE(tcs, ctor_and_getters);
374     ATF_ADD_TEST_CASE(tcs, find__ok);
375     ATF_ADD_TEST_CASE(tcs, find__missing);
376     ATF_ADD_TEST_CASE(tcs, test_cases__get);
377     ATF_ADD_TEST_CASE(tcs, test_cases__some);
378     ATF_ADD_TEST_CASE(tcs, test_cases__tester_fails);
379 
380     ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__copy);
381     ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__not_copy);
382 
383     ATF_ADD_TEST_CASE(tcs, output__no_test_cases);
384     ATF_ADD_TEST_CASE(tcs, output__some_test_cases);
385 }
386