1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of the QtTest module of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and The Qt Company. For licensing terms
14 ** and conditions see https://www.qt.io/terms-conditions. For further
15 ** information use the contact form at https://www.qt.io/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 3 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL3 included in the
21 ** packaging of this file. Please review the following information to
22 ** ensure the GNU Lesser General Public License version 3 requirements
23 ** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24 **
25 ** GNU General Public License Usage
26 ** Alternatively, this file may be used under the terms of the GNU
27 ** General Public License version 2.0 or (at your option) the GNU General
28 ** Public license version 3 or any later version approved by the KDE Free
29 ** Qt Foundation. The licenses are as published by the Free Software
30 ** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31 ** included in the packaging of this file. Please review the following
32 ** information to ensure the GNU General Public License requirements will
33 ** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34 ** https://www.gnu.org/licenses/gpl-3.0.html.
35 **
36 ** $QT_END_LICENSE$
37 **
38 ****************************************************************************/
39
40 #include <QtTest/private/qjunittestlogger_p.h>
41 #include <QtTest/private/qtestelement_p.h>
42 #include <QtTest/private/qtestjunitstreamer_p.h>
43 #include <QtTest/qtestcase.h>
44 #include <QtTest/private/qtestresult_p.h>
45 #include <QtTest/private/qbenchmark_p.h>
46
47 #ifdef min // windows.h without NOMINMAX is included by the benchmark headers.
48 # undef min
49 #endif
50 #ifdef max
51 # undef max
52 #endif
53
54 #include <QtCore/qlibraryinfo.h>
55
56 #include <string.h>
57
58 QT_BEGIN_NAMESPACE
59
QJUnitTestLogger(const char * filename)60 QJUnitTestLogger::QJUnitTestLogger(const char *filename)
61 : QAbstractTestLogger(filename)
62 {
63 }
64
~QJUnitTestLogger()65 QJUnitTestLogger::~QJUnitTestLogger()
66 {
67 delete currentLogElement;
68 delete logFormatter;
69 }
70
startLogging()71 void QJUnitTestLogger::startLogging()
72 {
73 QAbstractTestLogger::startLogging();
74
75 logFormatter = new QTestJUnitStreamer(this);
76 delete errorLogElement;
77 errorLogElement = new QTestElement(QTest::LET_SystemError);
78 }
79
stopLogging()80 void QJUnitTestLogger::stopLogging()
81 {
82 QTestElement *iterator = listOfTestcases;
83
84 char buf[10];
85
86 currentLogElement = new QTestElement(QTest::LET_TestSuite);
87 currentLogElement->addAttribute(QTest::AI_Name, QTestResult::currentTestObjectName());
88
89 qsnprintf(buf, sizeof(buf), "%i", testCounter);
90 currentLogElement->addAttribute(QTest::AI_Tests, buf);
91
92 qsnprintf(buf, sizeof(buf), "%i", failureCounter);
93 currentLogElement->addAttribute(QTest::AI_Failures, buf);
94
95 qsnprintf(buf, sizeof(buf), "%i", errorCounter);
96 currentLogElement->addAttribute(QTest::AI_Errors, buf);
97
98 QTestElement *property;
99 QTestElement *properties = new QTestElement(QTest::LET_Properties);
100
101 property = new QTestElement(QTest::LET_Property);
102 property->addAttribute(QTest::AI_Name, "QTestVersion");
103 property->addAttribute(QTest::AI_PropertyValue, QTEST_VERSION_STR);
104 properties->addLogElement(property);
105
106 property = new QTestElement(QTest::LET_Property);
107 property->addAttribute(QTest::AI_Name, "QtVersion");
108 property->addAttribute(QTest::AI_PropertyValue, qVersion());
109 properties->addLogElement(property);
110
111 property = new QTestElement(QTest::LET_Property);
112 property->addAttribute(QTest::AI_Name, "QtBuild");
113 property->addAttribute(QTest::AI_PropertyValue, QLibraryInfo::build());
114 properties->addLogElement(property);
115
116 currentLogElement->addLogElement(properties);
117
118 currentLogElement->addLogElement(iterator);
119
120 /* For correct indenting, make sure every testcase knows its parent */
121 QTestElement* testcase = iterator;
122 while (testcase) {
123 testcase->setParent(currentLogElement);
124 testcase = testcase->nextElement();
125 }
126
127 currentLogElement->addLogElement(errorLogElement);
128
129 QTestElement *it = currentLogElement;
130 logFormatter->output(it);
131
132 QAbstractTestLogger::stopLogging();
133 }
134
enterTestFunction(const char * function)135 void QJUnitTestLogger::enterTestFunction(const char *function)
136 {
137 currentLogElement = new QTestElement(QTest::LET_TestCase);
138 currentLogElement->addAttribute(QTest::AI_Name, function);
139 currentLogElement->addToList(&listOfTestcases);
140
141 ++testCounter;
142 }
143
leaveTestFunction()144 void QJUnitTestLogger::leaveTestFunction()
145 {
146 }
147
addIncident(IncidentTypes type,const char * description,const char * file,int line)148 void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
149 const char *file, int line)
150 {
151 const char *typeBuf = nullptr;
152 char buf[100];
153
154 switch (type) {
155 case QAbstractTestLogger::XPass:
156 ++failureCounter;
157 typeBuf = "xpass";
158 break;
159 case QAbstractTestLogger::Pass:
160 typeBuf = "pass";
161 break;
162 case QAbstractTestLogger::XFail:
163 typeBuf = "xfail";
164 break;
165 case QAbstractTestLogger::Fail:
166 ++failureCounter;
167 typeBuf = "fail";
168 break;
169 case QAbstractTestLogger::BlacklistedPass:
170 typeBuf = "bpass";
171 break;
172 case QAbstractTestLogger::BlacklistedFail:
173 ++failureCounter;
174 typeBuf = "bfail";
175 break;
176 case QAbstractTestLogger::BlacklistedXPass:
177 typeBuf = "bxpass";
178 break;
179 case QAbstractTestLogger::BlacklistedXFail:
180 ++failureCounter;
181 typeBuf = "bxfail";
182 break;
183 default:
184 typeBuf = "??????";
185 break;
186 }
187
188 if (type == QAbstractTestLogger::Fail || type == QAbstractTestLogger::XPass) {
189 QTestElement *failureElement = new QTestElement(QTest::LET_Failure);
190 failureElement->addAttribute(QTest::AI_Result, typeBuf);
191 if (file)
192 failureElement->addAttribute(QTest::AI_File, file);
193 else
194 failureElement->addAttribute(QTest::AI_File, "");
195 qsnprintf(buf, sizeof(buf), "%i", line);
196 failureElement->addAttribute(QTest::AI_Line, buf);
197 failureElement->addAttribute(QTest::AI_Description, description);
198 addTag(failureElement);
199 currentLogElement->addLogElement(failureElement);
200 }
201
202 /*
203 Only one result can be shown for the whole testfunction.
204 Check if we currently have a result, and if so, overwrite it
205 iff the new result is worse.
206 */
207 QTestElementAttribute* resultAttr =
208 const_cast<QTestElementAttribute*>(currentLogElement->attribute(QTest::AI_Result));
209 if (resultAttr) {
210 const char* oldResult = resultAttr->value();
211 bool overwrite = false;
212 if (!strcmp(oldResult, "pass")) {
213 overwrite = true;
214 }
215 else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
216 overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
217 || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
218 }
219 else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
220 overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
221 }
222 else if (!strcmp(oldResult, "xfail")) {
223 overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail);
224 }
225 else if (!strcmp(oldResult, "xpass")) {
226 overwrite = (type == QAbstractTestLogger::Fail);
227 }
228 if (overwrite) {
229 resultAttr->setPair(QTest::AI_Result, typeBuf);
230 }
231 }
232 else {
233 currentLogElement->addAttribute(QTest::AI_Result, typeBuf);
234 }
235
236 if (file)
237 currentLogElement->addAttribute(QTest::AI_File, file);
238 else
239 currentLogElement->addAttribute(QTest::AI_File, "");
240
241 qsnprintf(buf, sizeof(buf), "%i", line);
242 currentLogElement->addAttribute(QTest::AI_Line, buf);
243
244 /*
245 Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still
246 have some information about the expected failure.
247 */
248 if (type == QAbstractTestLogger::XFail) {
249 QJUnitTestLogger::addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
250 }
251 }
252
addBenchmarkResult(const QBenchmarkResult & result)253 void QJUnitTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
254 {
255 QTestElement *benchmarkElement = new QTestElement(QTest::LET_Benchmark);
256
257 benchmarkElement->addAttribute(
258 QTest::AI_Metric,
259 QTest::benchmarkMetricName(result.metric));
260 benchmarkElement->addAttribute(QTest::AI_Tag, result.context.tag.toUtf8().data());
261
262 const qreal valuePerIteration = qreal(result.value) / qreal(result.iterations);
263 benchmarkElement->addAttribute(QTest::AI_Value, QByteArray::number(valuePerIteration).constData());
264
265 char buf[100];
266 qsnprintf(buf, sizeof(buf), "%i", result.iterations);
267 benchmarkElement->addAttribute(QTest::AI_Iterations, buf);
268 currentLogElement->addLogElement(benchmarkElement);
269 }
270
addTag(QTestElement * element)271 void QJUnitTestLogger::addTag(QTestElement* element)
272 {
273 const char *tag = QTestResult::currentDataTag();
274 const char *gtag = QTestResult::currentGlobalDataTag();
275 const char *filler = (tag && gtag) ? ":" : "";
276 if ((!tag || !tag[0]) && (!gtag || !gtag[0])) {
277 return;
278 }
279
280 if (!tag) {
281 tag = "";
282 }
283 if (!gtag) {
284 gtag = "";
285 }
286
287 QTestCharBuffer buf;
288 QTest::qt_asprintf(&buf, "%s%s%s", gtag, filler, tag);
289 element->addAttribute(QTest::AI_Tag, buf.constData());
290 }
291
addMessage(MessageTypes type,const QString & message,const char * file,int line)292 void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
293 {
294 QTestElement *errorElement = new QTestElement(QTest::LET_Error);
295 const char *typeBuf = nullptr;
296
297 switch (type) {
298 case QAbstractTestLogger::Warn:
299 typeBuf = "warn";
300 break;
301 case QAbstractTestLogger::QSystem:
302 typeBuf = "system";
303 break;
304 case QAbstractTestLogger::QDebug:
305 typeBuf = "qdebug";
306 break;
307 case QAbstractTestLogger::QInfo:
308 typeBuf = "qinfo";
309 break;
310 case QAbstractTestLogger::QWarning:
311 typeBuf = "qwarn";
312 break;
313 case QAbstractTestLogger::QFatal:
314 typeBuf = "qfatal";
315 break;
316 case QAbstractTestLogger::Skip:
317 typeBuf = "skip";
318 break;
319 case QAbstractTestLogger::Info:
320 typeBuf = "info";
321 break;
322 default:
323 typeBuf = "??????";
324 break;
325 }
326
327 errorElement->addAttribute(QTest::AI_Type, typeBuf);
328 errorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
329 addTag(errorElement);
330
331 if (file)
332 errorElement->addAttribute(QTest::AI_File, file);
333 else
334 errorElement->addAttribute(QTest::AI_File, "");
335
336 char buf[100];
337 qsnprintf(buf, sizeof(buf), "%i", line);
338 errorElement->addAttribute(QTest::AI_Line, buf);
339
340 currentLogElement->addLogElement(errorElement);
341 ++errorCounter;
342
343 // Also add the message to the system error log (i.e. stderr), if one exists
344 if (errorLogElement) {
345 QTestElement *systemErrorElement = new QTestElement(QTest::LET_Error);
346 systemErrorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
347 errorLogElement->addLogElement(systemErrorElement);
348 }
349 }
350
351 QT_END_NAMESPACE
352
353