1 // Copyright 2012 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "engine/testers.hpp"
30
31 extern "C" {
32 #include <dirent.h>
33 #include <regex.h>
34 }
35
36 #include <cerrno>
37 #include <cstring>
38 #include <iostream>
39 #include <map>
40 #include <string>
41
42 #include "engine/exceptions.hpp"
43 #include "utils/env.hpp"
44 #include "utils/format/macros.hpp"
45 #include "utils/fs/operations.hpp"
46 #include "utils/fs/path.hpp"
47 #include "utils/logging/macros.hpp"
48 #include "utils/optional.ipp"
49 #include "utils/passwd.hpp"
50 #include "utils/process/child.ipp"
51 #include "utils/process/status.hpp"
52 #include "utils/stream.hpp"
53
54 namespace datetime = utils::datetime;
55 namespace fs = utils::fs;
56 namespace logging = utils::logging;
57 namespace passwd = utils::passwd;
58 namespace process = utils::process;
59
60 using utils::none;
61 using utils::optional;
62
63
64 namespace {
65
66
67 /// Mapping of interface names to tester binaries.
68 typedef std::map< std::string, std::string > testers_map;
69
70
71 /// Collection of known-good interface to tester mappings.
72 static testers_map interfaces_to_testers;
73
74
75 /// Drops the trailing newline in a string and replaces others with a literal.
76 ///
77 /// \param input The string in which to perform the replacements.
78 ///
79 /// \return The modified string.
80 static std::string
replace_newlines(const std::string input)81 replace_newlines(const std::string input)
82 {
83 std::string output = input;
84
85 while (output.length() > 0 && output[output.length() - 1] == '\n') {
86 output.erase(output.end() - 1);
87 }
88
89 std::string::size_type newline = output.find('\n', 0);
90 while (newline != std::string::npos) {
91 output.replace(newline, 1, "<<NEWLINE>>");
92 newline = output.find('\n', newline + 1);
93 }
94
95 return output;
96 }
97
98
99 /// RAII pattern to invoke a release method on destruction.
100 ///
101 /// \todo The existence of this class here is a hack. We should either
102 /// generalize the class and use it wherever we need release on destruction
103 /// semantics, or we should have proper abstractions for the objects below that
104 /// use this class.
105 ///
106 /// \tparam Object The type of the object to be released. Not a pointer.
107 /// \tparam ReturnType The return type of the release method.
108 template< typename Object, typename ReturnType >
109 class object_releaser {
110 /// Pointer to the object being managed.
111 Object* _object;
112
113 /// Release hook.
114 ReturnType (*_free_hook)(Object*);
115
116 public:
117 /// Constructor.
118 ///
119 /// \param object Pointer to the object being managed.
120 /// \param free_hook Release hook.
object_releaser(Object * object,ReturnType (* free_hook)(Object *))121 object_releaser(Object* object, ReturnType (*free_hook)(Object*)) :
122 _object(object), _free_hook(free_hook)
123 {
124 }
125
126 /// Destructor.
~object_releaser(void)127 ~object_releaser(void)
128 {
129 _free_hook(_object);
130 }
131 };
132
133
134 /// Finds all available testers and caches their data.
135 ///
136 /// \param [out] testers Map into which to store the list of available testers.
137 static void
load_testers(testers_map & testers)138 load_testers(testers_map& testers)
139 {
140 PRE(testers.empty());
141
142 const fs::path raw_testersdir(utils::getenv_with_default(
143 "KYUA_TESTERSDIR", KYUA_TESTERSDIR));
144 const fs::path testersdir = raw_testersdir.is_absolute() ?
145 raw_testersdir : raw_testersdir.to_absolute();
146
147 ::DIR* dir = ::opendir(testersdir.c_str());
148 if (dir == NULL) {
149 const int original_errno = errno;
150 LW(F("Failed to open testers dir %s: %s") % testersdir %
151 strerror(original_errno));
152 return; // No testers available in the given location.
153 }
154 const object_releaser< ::DIR, int > dir_releaser(dir, ::closedir);
155
156 ::regex_t preg;
157 if (::regcomp(&preg, "^kyua-(.+)-tester$", REG_EXTENDED) != 0)
158 throw engine::error("Failed to compile regular expression");
159 const object_releaser< ::regex_t, void > preg_releaser(&preg, ::regfree);
160
161 ::dirent* de;
162 while ((de = readdir(dir)) != NULL) {
163 ::regmatch_t matches[2];
164 const int ret = ::regexec(&preg, de->d_name, 2, matches, 0);
165 if (ret == 0) {
166 const std::string interface(de->d_name + matches[1].rm_so,
167 matches[1].rm_eo - matches[1].rm_so);
168 const fs::path path = testersdir / de->d_name;
169 LI(F("Found tester for interface %s in %s") % interface % path);
170 INV(path.is_absolute());
171 testers[interface] = path.str();
172 } else if (ret == REG_NOMATCH) {
173 // Not a tester; skip.
174 } else {
175 throw engine::error("Failed to match regular expression");
176 }
177 }
178 }
179
180
181 } // anonymous namespace
182
183
184 /// Returns the path to a tester binary.
185 ///
186 /// \param interface Name of the interface of the tester being looked for.
187 ///
188 /// \return Absolute path to the tester.
189 fs::path
tester_path(const std::string & interface)190 engine::tester_path(const std::string& interface)
191 {
192 if (interfaces_to_testers.empty())
193 load_testers(interfaces_to_testers);
194
195 const testers_map::const_iterator iter = interfaces_to_testers.find(
196 interface);
197 if (iter == interfaces_to_testers.end())
198 throw engine::error("Unknown interface " + interface);
199
200 const fs::path path((*iter).second);
201 INV(path.is_absolute());
202 return path;
203 }
204
205
206 /// Constructs a tester.
207 ///
208 /// \param interface Name of the interface to use.
209 /// \param unprivileged_user If not none, the user to switch to when running
210 /// the tester.
211 /// \param timeout If not none, the timeout to pass to the tester.
tester(const std::string & interface,const optional<passwd::user> & unprivileged_user,const optional<datetime::delta> & timeout)212 engine::tester::tester(const std::string& interface,
213 const optional< passwd::user >& unprivileged_user,
214 const optional< datetime::delta >& timeout) :
215 _interface(interface)
216 {
217 if (unprivileged_user) {
218 _common_args.push_back(F("-u%s") % unprivileged_user.get().uid);
219 _common_args.push_back(F("-g%s") % unprivileged_user.get().gid);
220 }
221 if (timeout) {
222 PRE(timeout.get().useconds == 0);
223 _common_args.push_back(F("-t%s") % timeout.get().seconds);
224 }
225 }
226
227
228 /// Destructor.
~tester(void)229 engine::tester::~tester(void)
230 {
231 }
232
233
234 /// Executes a list operation on a test program.
235 ///
236 /// \param program Path to the test program.
237 ///
238 /// \return The output of the tester, which represents a valid list of test
239 /// cases.
240 ///
241 /// \throw error If the tester returns with an unsuccessful exit code.
242 std::string
list(const fs::path & program) const243 engine::tester::list(const fs::path& program) const
244 {
245 std::vector< std::string > args = _common_args;
246 args.push_back("list");
247 args.push_back(program.str());
248
249 const fs::path tester_path = engine::tester_path(_interface);
250 std::unique_ptr< process::child > child = process::child::spawn_capture(
251 tester_path, args);
252
253 const std::string output = utils::read_stream(child->output());
254
255 const process::status status = child->wait();
256 if (!status.exited() || status.exitstatus() != EXIT_SUCCESS)
257 throw engine::error("Tester did not exit cleanly: " +
258 replace_newlines(output));
259 return output;
260 }
261
262
263 /// Executes a test operation on a test case.
264 ///
265 /// \param program Path to the test program.
266 /// \param test_case_name Name of the test case to execute.
267 /// \param result_file Path to the file in which to leave the result of the
268 /// tester invocation.
269 /// \param stdout_file Path to the file in which to store the stdout.
270 /// \param stderr_file Path to the file in which to store the stderr.
271 /// \param vars Collection of configuration variables.
272 ///
273 /// \throw error If the tester returns with an unsuccessful exit code.
274 void
test(const fs::path & program,const std::string & test_case_name,const fs::path & result_file,const fs::path & stdout_file,const fs::path & stderr_file,const std::map<std::string,std::string> & vars) const275 engine::tester::test(const fs::path& program, const std::string& test_case_name,
276 const fs::path& result_file, const fs::path& stdout_file,
277 const fs::path& stderr_file,
278 const std::map< std::string, std::string >& vars) const
279 {
280 std::vector< std::string > args = _common_args;
281 args.push_back("test");
282 for (std::map< std::string, std::string >::const_iterator i = vars.begin();
283 i != vars.end(); ++i) {
284 args.push_back(F("-v%s=%s") % (*i).first % (*i).second);
285 }
286 args.push_back(program.str());
287 args.push_back(test_case_name);
288 args.push_back(result_file.str());
289
290 const fs::path tester_path = engine::tester_path(_interface);
291 std::unique_ptr< process::child > child = process::child::spawn_files(
292 tester_path, args, stdout_file, stderr_file);
293 const process::status status = child->wait();
294
295 if (status.exited()) {
296 if (status.exitstatus() == EXIT_SUCCESS) {
297 // OK; the tester exited cleanly.
298 } else if (status.exitstatus() == EXIT_FAILURE) {
299 // OK; the tester reported that the test itself failed and we have
300 // the result file to indicate this.
301 } else {
302 throw engine::error(F("Tester failed with code %s; this is a bug") %
303 status.exitstatus());
304 }
305 } else {
306 INV(status.signaled());
307 throw engine::error(F("Tester received signal %s; this is a bug") %
308 status.termsig());
309 }
310 }
311