1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of the QtTest module of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and The Qt Company. For licensing terms
14 ** and conditions see https://www.qt.io/terms-conditions. For further
15 ** information use the contact form at https://www.qt.io/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 3 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL3 included in the
21 ** packaging of this file. Please review the following information to
22 ** ensure the GNU Lesser General Public License version 3 requirements
23 ** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24 **
25 ** GNU General Public License Usage
26 ** Alternatively, this file may be used under the terms of the GNU
27 ** General Public License version 2.0 or (at your option) the GNU General
28 ** Public license version 3 or any later version approved by the KDE Free
29 ** Qt Foundation. The licenses are as published by the Free Software
30 ** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31 ** included in the packaging of this file. Please review the following
32 ** information to ensure the GNU General Public License requirements will
33 ** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34 ** https://www.gnu.org/licenses/gpl-3.0.html.
35 **
36 ** $QT_END_LICENSE$
37 **
38 ****************************************************************************/
39 #include "qtestblacklist_p.h"
40 #include "qtestresult_p.h"
41
42 #include <QtTest/qtestcase.h>
43 #include <QtCore/qbytearray.h>
44 #include <QtCore/qfile.h>
45 #include <QtCore/qset.h>
46 #include <QtCore/qcoreapplication.h>
47 #include <QtCore/qvariant.h>
48 #include <QtCore/QSysInfo>
49
50 #include <set>
51
52 QT_BEGIN_NAMESPACE
53
54 /*
55 The BLACKLIST file format is a grouped listing of keywords.
56
57 Blank lines and everything after # is simply ignored. An initial #-line
58 referring to this documentation is kind to readers. Comments can also be used
59 to indicate the reasons for ignoring particular cases.
60
61 The key "ci" applies only when run by COIN. Other keys name platforms,
62 operating systems, distributions, tool-chains or architectures; a ! prefix
63 reverses what it checks. A version, joined to a key (at present, only for
64 distributions and for msvc) with a hyphen, limits the key to the specific
65 version. A keyword line matches if every key on it applies to the present
66 run. Successive lines are alternate conditions for ignoring a test.
67
68 Ungrouped lines at the beginning of a file apply to the whole testcase.
69 A group starts with a [square-bracketed] identification of a test function,
70 optionally with (after a colon, the name of) a specific data set, to ignore.
71 Subsequent lines give conditions for ignoring this test.
72
73 # See qtbase/src/testlib/qtestblacklist.cpp for format
74 # Test doesn't work on QNX at all
75 qnx
76
77 # QTBUG-12345
78 [testFunction]
79 linux
80 windows 64bit
81
82 # Flaky in COIN on macOS, not reproducible by developers
83 [testSlowly]
84 ci osx
85
86 # Needs basic C++11 support
87 [testfunction2:testData]
88 msvc-2010
89
90 QML test functions are identified using the following format:
91
92 <TestCase name>::<function name>:<data tag>
93
94 For example, to blacklist a QML test on RHEL 7.6:
95
96 # QTBUG-12345
97 [Button::test_display:TextOnly]
98 ci rhel-7.6
99
100 Keys are lower-case. Distribution name and version are supported if
101 QSysInfo's productType() and productVersion() return them.
102
103 Keys can be added via the space-separated QTEST_ENVIRONMENT
104 environment variable:
105
106 QTEST_ENVIRONMENT=ci ./tst_stuff
107
108 This can be used to "mock" a test environment. In the example above,
109 we add "ci" to the list of keys for the test environment, making it
110 possible to test BLACKLIST files that blacklist tests in a CI environment.
111
112 The other known keys are listed below:
113 */
114
keywords()115 static QSet<QByteArray> keywords()
116 {
117 // this list can be extended with new keywords as required
118 QSet<QByteArray> set = QSet<QByteArray>()
119 << "*"
120 #ifdef Q_OS_LINUX
121 << "linux"
122 #endif
123 #ifdef Q_OS_MACOS
124 << "osx"
125 << "macos"
126 #endif
127 #if defined(Q_OS_WIN) && !defined(Q_OS_WINRT)
128 << "windows"
129 #endif
130 #ifdef Q_OS_IOS
131 << "ios"
132 #endif
133 #ifdef Q_OS_TVOS
134 << "tvos"
135 #endif
136 #ifdef Q_OS_WATCHOS
137 << "watchos"
138 #endif
139 #ifdef Q_OS_ANDROID
140 << "android"
141 #endif
142 #ifdef Q_OS_QNX
143 << "qnx"
144 #endif
145 #ifdef Q_OS_WINRT
146 << "winrt"
147 #endif
148
149 #if QT_POINTER_SIZE == 8
150 << "64bit"
151 #else
152 << "32bit"
153 #endif
154
155 #ifdef Q_CC_GNU
156 << "gcc"
157 #endif
158 #ifdef Q_CC_CLANG
159 << "clang"
160 #endif
161 #ifdef Q_CC_MSVC
162 << "msvc"
163 # if _MSC_VER <= 1600
164 << "msvc-2010"
165 # elif _MSC_VER <= 1700
166 << "msvc-2012"
167 # elif _MSC_VER <= 1800
168 << "msvc-2013"
169 # elif _MSC_VER <= 1900
170 << "msvc-2015"
171 # elif _MSC_VER <= 1916
172 << "msvc-2017"
173 # else
174 << "msvc-2019"
175 # endif
176 #endif
177
178 #ifdef Q_PROCESSOR_X86
179 << "x86"
180 #endif
181 #ifdef Q_PROCESSOR_ARM
182 << "arm"
183 #endif
184
185 #ifdef QT_BUILD_INTERNAL
186 << "developer-build"
187 #endif
188 ;
189
190 #if QT_CONFIG(properties)
191 QCoreApplication *app = QCoreApplication::instance();
192 if (app) {
193 const QVariant platformName = app->property("platformName");
194 if (platformName.isValid())
195 set << platformName.toByteArray();
196 }
197 #endif
198
199 return set;
200 }
201
activeConditions()202 static QSet<QByteArray> activeConditions()
203 {
204 QSet<QByteArray> result = keywords();
205
206 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
207 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
208 if (!distributionName.isEmpty()) {
209 if (result.find(distributionName) == result.end())
210 result.insert(distributionName);
211 if (!distributionRelease.isEmpty()) {
212 QByteArray versioned = distributionName + "-" + distributionRelease;
213 if (result.find(versioned) == result.end())
214 result.insert(versioned);
215 }
216 }
217
218 if (qEnvironmentVariableIsSet("QTEST_ENVIRONMENT")) {
219 for (const QByteArray &k : qgetenv("QTEST_ENVIRONMENT").split(' '))
220 result.insert(k);
221 }
222
223 return result;
224 }
225
checkCondition(const QByteArray & condition)226 static bool checkCondition(const QByteArray &condition)
227 {
228 static const QSet<QByteArray> matchedConditions = activeConditions();
229 QList<QByteArray> conds = condition.split(' ');
230
231 for (QByteArray c : conds) {
232 bool result = c.startsWith('!');
233 if (result)
234 c.remove(0, 1);
235
236 result ^= matchedConditions.contains(c);
237 if (!result)
238 return false;
239 }
240 return true;
241 }
242
243 static bool ignoreAll = false;
244 static std::set<QByteArray> *ignoredTests = nullptr;
245
246 namespace QTestPrivate {
247
parseBlackList()248 void parseBlackList()
249 {
250 QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
251 if (filename.isEmpty())
252 return;
253 QFile ignored(filename);
254 if (!ignored.open(QIODevice::ReadOnly))
255 return;
256
257 QByteArray function;
258
259 while (!ignored.atEnd()) {
260 QByteArray line = ignored.readLine();
261 const int commentPosition = line.indexOf('#');
262 if (commentPosition >= 0)
263 line.truncate(commentPosition);
264 line = line.simplified();
265 if (line.isEmpty())
266 continue;
267 if (line.startsWith('[')) {
268 function = line.mid(1, line.length() - 2);
269 continue;
270 }
271 bool condition = checkCondition(line);
272 if (condition) {
273 if (!function.size()) {
274 ignoreAll = true;
275 } else {
276 if (!ignoredTests)
277 ignoredTests = new std::set<QByteArray>;
278 ignoredTests->insert(function);
279 }
280 }
281 }
282 }
283
checkBlackLists(const char * slot,const char * data)284 void checkBlackLists(const char *slot, const char *data)
285 {
286 bool ignore = ignoreAll;
287
288 if (!ignore && ignoredTests) {
289 QByteArray s = slot;
290 ignore = (ignoredTests->find(s) != ignoredTests->end());
291 if (!ignore && data) {
292 s += ':';
293 s += data;
294 ignore = (ignoredTests->find(s) != ignoredTests->end());
295 }
296 }
297
298 QTestResult::setBlacklistCurrentTest(ignore);
299 }
300
301 } // QTestPrivate
302
303 QT_END_NAMESPACE
304