1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Test log compare utility.
22 *//*--------------------------------------------------------------------*/
23
24 #include "xeTestLogParser.hpp"
25 #include "xeTestResultParser.hpp"
26 #include "deFilePath.hpp"
27 #include "deString.h"
28 #include "deThread.hpp"
29 #include "deCommandLine.hpp"
30
31 #include <vector>
32 #include <string>
33 #include <cstdio>
34 #include <cstdlib>
35 #include <fstream>
36 #include <iostream>
37 #include <set>
38 #include <map>
39
40 using std::vector;
41 using std::string;
42 using std::set;
43 using std::map;
44
45 enum OutputMode
46 {
47 OUTPUTMODE_ALL = 0,
48 OUTPUTMODE_DIFF,
49
50 OUTPUTMODE_LAST
51 };
52
53 enum OutputFormat
54 {
55 OUTPUTFORMAT_TEXT = 0,
56 OUTPUTFORMAT_CSV,
57
58 OUTPUTFORMAT_LAST
59 };
60
61 enum OutputValue
62 {
63 OUTPUTVALUE_STATUS_CODE = 0,
64 OUTPUTVALUE_STATUS_DETAILS,
65
66 OUTPUTVALUE_LAST
67 };
68
69 namespace opt
70 {
71
72 DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode);
73 DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat);
74 DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue);
75
registerOptions(de::cmdline::Parser & parser)76 static void registerOptions (de::cmdline::Parser& parser)
77 {
78 using de::cmdline::Option;
79 using de::cmdline::NamedValue;
80
81 static const NamedValue<OutputMode> s_outputModes[] =
82 {
83 { "all", OUTPUTMODE_ALL },
84 { "diff", OUTPUTMODE_DIFF }
85 };
86 static const NamedValue<OutputFormat> s_outputFormats[] =
87 {
88 { "text", OUTPUTFORMAT_TEXT },
89 { "csv", OUTPUTFORMAT_CSV }
90 };
91 static const NamedValue<OutputValue> s_outputValues[] =
92 {
93 { "code", OUTPUTVALUE_STATUS_CODE },
94 { "details", OUTPUTVALUE_STATUS_DETAILS }
95 };
96
97 parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv")
98 << Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all")
99 << Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code");
100 }
101
102 } // opt
103
104 struct CommandLine
105 {
CommandLineCommandLine106 CommandLine (void)
107 : outMode (OUTPUTMODE_ALL)
108 , outFormat (OUTPUTFORMAT_CSV)
109 , outValue (OUTPUTVALUE_STATUS_CODE)
110 {
111 }
112
113 OutputMode outMode;
114 OutputFormat outFormat;
115 OutputValue outValue;
116 vector<string> filenames;
117 };
118
119 struct ShortBatchResult
120 {
121 vector<xe::TestCaseResultHeader> resultHeaders;
122 map<string, int> resultMap;
123 };
124
125 class ShortResultHandler : public xe::TestLogHandler
126 {
127 public:
ShortResultHandler(ShortBatchResult & result)128 ShortResultHandler (ShortBatchResult& result)
129 : m_result(result)
130 {
131 }
132
setSessionInfo(const xe::SessionInfo &)133 void setSessionInfo (const xe::SessionInfo&)
134 {
135 // Ignored.
136 }
137
startTestCaseResult(const char * casePath)138 xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
139 {
140 return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
141 }
142
testCaseResultUpdated(const xe::TestCaseResultPtr &)143 void testCaseResultUpdated (const xe::TestCaseResultPtr&)
144 {
145 // Ignored.
146 }
147
testCaseResultComplete(const xe::TestCaseResultPtr & caseData)148 void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
149 {
150 xe::TestCaseResultHeader header;
151 int caseNdx = (int)m_result.resultHeaders.size();
152
153 header.casePath = caseData->getTestCasePath();
154 header.caseType = xe::TESTCASETYPE_SELF_VALIDATE;
155 header.statusCode = caseData->getStatusCode();
156 header.statusDetails = caseData->getStatusDetails();
157
158 if (header.statusCode == xe::TESTSTATUSCODE_LAST)
159 {
160 xe::TestCaseResult fullResult;
161
162 xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
163
164 header = xe::TestCaseResultHeader(fullResult);
165 }
166
167 // Insert into result list & map.
168 m_result.resultHeaders.push_back(header);
169 m_result.resultMap[header.casePath] = caseNdx;
170 }
171
172 private:
173 ShortBatchResult& m_result;
174 xe::TestResultParser m_testResultParser;
175 };
176
readLogFile(ShortBatchResult & batchResult,const char * filename)177 static void readLogFile (ShortBatchResult& batchResult, const char* filename)
178 {
179 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
180 ShortResultHandler resultHandler (batchResult);
181 xe::TestLogParser parser (&resultHandler);
182 deUint8 buf [1024];
183 int numRead = 0;
184
185 for (;;)
186 {
187 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
188 numRead = (int)in.gcount();
189
190 if (numRead <= 0)
191 break;
192
193 parser.parse(&buf[0], numRead);
194 }
195
196 in.close();
197 }
198
199 class LogFileReader : public de::Thread
200 {
201 public:
LogFileReader(ShortBatchResult & batchResult,const char * filename)202 LogFileReader (ShortBatchResult& batchResult, const char* filename)
203 : m_batchResult (batchResult)
204 , m_filename (filename)
205 {
206 }
207
run(void)208 void run (void)
209 {
210 readLogFile(m_batchResult, m_filename.c_str());
211 }
212
213 private:
214 ShortBatchResult& m_batchResult;
215 std::string m_filename;
216 };
217
computeCaseList(vector<string> & cases,const vector<ShortBatchResult> & batchResults)218 static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
219 {
220 // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
221 set<string> addedCases;
222
223 for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
224 {
225 for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
226 {
227 if (addedCases.find(caseIter->casePath) == addedCases.end())
228 {
229 cases.push_back(caseIter->casePath);
230 addedCases.insert(caseIter->casePath);
231 }
232 }
233 }
234 }
235
getTestResultHeaders(vector<xe::TestCaseResultHeader> & headers,const vector<ShortBatchResult> & batchResults,const char * casePath)236 static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
237 {
238 headers.resize(batchResults.size());
239
240 for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
241 {
242 const ShortBatchResult& batchResult = batchResults[ndx];
243 map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath);
244
245 if (resultPos != batchResult.resultMap.end())
246 headers[ndx] = batchResult.resultHeaders[resultPos->second];
247 else
248 {
249 headers[ndx].casePath = casePath;
250 headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE;
251 headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST;
252 }
253 }
254 }
255
getStatusCodeName(xe::TestStatusCode code)256 static const char* getStatusCodeName (xe::TestStatusCode code)
257 {
258 if (code == xe::TESTSTATUSCODE_LAST)
259 return "Missing";
260 else
261 return xe::getTestStatusCodeName(code);
262 }
263
runCompare(const CommandLine & cmdLine,std::ostream & dst)264 static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
265 {
266 vector<ShortBatchResult> results;
267 vector<string> batchNames;
268 bool compareOk = true;
269
270 XE_CHECK(!cmdLine.filenames.empty());
271
272 try
273 {
274 // Read in batch results
275 results.resize(cmdLine.filenames.size());
276 {
277 std::vector<de::SharedPtr<LogFileReader> > readers;
278
279 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
280 {
281 readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
282 readers.back()->start();
283 }
284
285 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
286 {
287 readers[ndx]->join();
288
289 // Use file name as batch name.
290 batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
291 }
292 }
293
294 // Compute unified case list.
295 vector<string> caseList;
296 computeCaseList(caseList, results);
297
298 // Stats.
299 int numCases = (int)caseList.size();
300 int numEqual = 0;
301
302 if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
303 {
304 dst << "TestCasePath";
305 for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
306 dst << "," << *nameIter;
307 dst << "\n";
308 }
309
310 // Compare cases.
311 for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
312 {
313 const string& caseName = *caseIter;
314 vector<xe::TestCaseResultHeader> headers;
315 bool allEqual = true;
316
317 getTestResultHeaders(headers, results, caseName.c_str());
318
319 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
320 {
321 if (iter->statusCode != headers[0].statusCode)
322 {
323 allEqual = false;
324 break;
325 }
326 }
327
328 if (allEqual)
329 numEqual += 1;
330
331 if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
332 {
333 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
334 {
335 dst << caseName << "\n";
336 for (int ndx = 0; ndx < (int)headers.size(); ndx++)
337 dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
338 dst << "\n";
339 }
340 else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
341 {
342 dst << caseName;
343 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
344 dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
345 dst << "\n";
346 }
347 }
348 }
349
350 compareOk = numEqual == numCases;
351
352 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
353 {
354 dst << " " << numEqual << " / " << numCases << " test case results match.\n";
355 dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
356 }
357 }
358 catch (const std::exception& e)
359 {
360 printf("%s\n", e.what());
361 compareOk = false;
362 }
363
364 return compareOk;
365 }
366
parseCommandLine(CommandLine & cmdLine,int argc,const char * const * argv)367 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
368 {
369 de::cmdline::Parser parser;
370 de::cmdline::CommandLine opts;
371
372 XE_CHECK(argc >= 1);
373
374 opt::registerOptions(parser);
375
376 if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) ||
377 opts.getArgs().empty())
378 {
379 std::cout << argv[0] << ": [options] [filenames]\n";
380 parser.help(std::cout);
381 return false;
382 }
383
384 cmdLine.outFormat = opts.getOption<opt::OutFormat>();
385 cmdLine.outMode = opts.getOption<opt::OutMode>();
386 cmdLine.outValue = opts.getOption<opt::OutValue>();
387 cmdLine.filenames = opts.getArgs();
388
389 return true;
390 }
391
main(int argc,const char * const * argv)392 int main (int argc, const char* const* argv)
393 {
394 CommandLine cmdLine;
395
396 if (!parseCommandLine(cmdLine, argc, argv))
397 return -1;
398
399 try
400 {
401 bool compareOk = runCompare(cmdLine, std::cout);
402 return compareOk ? 0 : -1;
403 }
404 catch (const std::exception& e)
405 {
406 printf("FATAL ERROR: %s\n", e.what());
407 return -1;
408 }
409 }
410