1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3#include "qtestblacklist_p.h"
4#include "qtestresult_p.h"
5
6#include <QtTest/qtestcase.h>
7#include <QtCore/qbytearray.h>
8#include <QtCore/qfile.h>
9#include <QtCore/qset.h>
10#include <QtCore/qcoreapplication.h>
11#include <QtCore/qvariant.h>
12#include <QtCore/QSysInfo>
13#include <QtCore/QOperatingSystemVersion>
14
15#include <set>
16
17QT_BEGIN_NAMESPACE
18
19using namespace Qt::StringLiterals;
20
21/*
22 The BLACKLIST file format is a grouped listing of keywords.
23
24 Blank lines and everything after # is simply ignored. An initial #-line
25 referring to this documentation is kind to readers. Comments can also be used
26 to indicate the reasons for ignoring particular cases.
27
28 Each blacklist line is interpreted as a list of keywords in an AND-relationship.
29 To blacklist a test for multiple platforms (OR-relationship), use separate lines.
30
31 The key "ci" applies only when run by COIN. The key "cmake" applies when Qt
32 is built using CMake. Other keys name platforms, operating systems,
33 distributions, tool-chains or architectures; a ! prefix reverses what it
34 checks. A version, joined to a key (at present, only for distributions and
35 for msvc) with a hyphen, limits the key to the specific version. A keyword
36 line matches if every key on it applies to the present run. Successive lines
37 are alternate conditions for ignoring a test.
38
39 Ungrouped lines at the beginning of a file apply to the whole testcase. A
40 group starts with a [square-bracketed] identification of a test function to
41 ignore. For data-driven tests, this identification can be narrowed by the
42 inclusion of global and local data row tags, separated from the function name
43 and each other by colons. If both global and function-specific data rows tags
44 are supplied, the global one comes first (as in the tag reported in test
45 output, albeit in parentheses after the function name). Even when a test does
46 have global and local data tags, you can omit either or both. (If a global
47 data row's name coincides with that of a local data row, some unintended
48 matches may result; try to keep your data-row tags distinct.)
49
50 Subsequent lines give conditions for ignoring this test. You need at least
51 one or the group has no effect.
52
53 # See qtbase/src/testlib/qtestblacklist.cpp for format
54 # Test doesn't work on QNX at all
55 qnx
56
57 # QTBUG-12345
58 [testFunction]
59 linux
60 windows 64bit
61
62 # Flaky in COIN on macOS, not reproducible by developers
63 [testSlowly]
64 macos ci
65
66 # Needs basic C++11 support
67 [testfunction2:testData]
68 msvc-2010
69
70 [getFile:withProxy SSL:localhost]
71 android
72
73 QML test functions are identified using the following format:
74
75 <TestCase name>::<function name>:<data tag>
76
77 For example, to blacklist a QML test on RHEL 7.6:
78
79 # QTBUG-12345
80 [Button::test_display:TextOnly]
81 ci rhel-7.6
82
83 Keys are lower-case. Distribution name and version are supported if
84 QSysInfo's productType() and productVersion() return them.
85
86 Keys can be added via the space-separated QTEST_ENVIRONMENT
87 environment variable:
88
89 QTEST_ENVIRONMENT=ci ./tst_stuff
90
91 This can be used to "mock" a test environment. In the example above,
92 we add "ci" to the list of keys for the test environment, making it
93 possible to test BLACKLIST files that blacklist tests in a CI environment.
94
95 The other known keys are listed below:
96*/
97
98static QSet<QByteArray> keywords()
99{
100 // this list can be extended with new keywords as required
101 QSet<QByteArray> set = QSet<QByteArray>()
102 << "*"
103#ifdef Q_OS_LINUX
104 << "linux"
105#endif
106#ifdef Q_OS_MACOS
107 << "osx"
108 << "macos"
109#endif
110#if defined(Q_OS_WIN)
111 << "windows"
112#endif
113#ifdef Q_OS_IOS
114 << "ios"
115#endif
116#ifdef Q_OS_TVOS
117 << "tvos"
118#endif
119#ifdef Q_OS_WATCHOS
120 << "watchos"
121#endif
122#ifdef Q_OS_ANDROID
123 << "android"
124#endif
125#ifdef Q_OS_QNX
126 << "qnx"
127#endif
128#ifdef Q_OS_WEBOS
129 << "webos"
130#endif
131
132#if QT_POINTER_SIZE == 8
133 << "64bit"
134#else
135 << "32bit"
136#endif
137
138#ifdef Q_CC_GNU
139 << "gcc"
140#endif
141#ifdef Q_CC_CLANG
142 << "clang"
143#endif
144#ifdef Q_CC_MSVC
145 << "msvc"
146# if _MSC_VER <= 1600
147 << "msvc-2010"
148# elif _MSC_VER <= 1700
149 << "msvc-2012"
150# elif _MSC_VER <= 1800
151 << "msvc-2013"
152# elif _MSC_VER <= 1900
153 << "msvc-2015"
154# elif _MSC_VER <= 1916
155 << "msvc-2017"
156# elif _MSC_VER <= 1929
157 << "msvc-2019"
158# else
159 << "msvc-2022"
160# endif
161#endif
162
163#ifdef Q_PROCESSOR_X86
164 << "x86"
165#endif
166#ifdef Q_PROCESSOR_ARM
167 << "arm"
168#endif
169
170#ifdef QT_BUILD_INTERNAL
171 << "developer-build"
172#endif
173
174 << "cmake"
175 ;
176
177 QCoreApplication *app = QCoreApplication::instance();
178 if (app) {
179 const QVariant platformName = app->property(name: "platformName");
180 if (platformName.isValid())
181 set << platformName.toByteArray();
182 }
183
184 return set;
185}
186
187static QSet<QByteArray> activeConditions()
188{
189 QSet<QByteArray> result = keywords();
190
191 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
192 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
193 if (!distributionName.isEmpty()) {
194 if (result.find(value: distributionName) == result.end())
195 result.insert(value: distributionName);
196 // backwards compatibility with Qt 5
197 if (distributionName == "macos") {
198 if (result.find(value: distributionName) == result.end())
199 result.insert(value: "osx");
200 const auto version = QOperatingSystemVersion::current();
201 if (version.majorVersion() >= 11)
202 distributionRelease = QByteArray::number(version.majorVersion());
203 }
204 if (!distributionRelease.isEmpty()) {
205 QByteArray versioned = distributionName + "-" + distributionRelease;
206 if (result.find(value: versioned) == result.end())
207 result.insert(value: versioned);
208 if (distributionName == "macos") {
209 QByteArray versioned = "osx-" + distributionRelease;
210 if (result.find(value: versioned) == result.end())
211 result.insert(value: versioned);
212 }
213 }
214 }
215
216 if (qEnvironmentVariableIsSet(varName: "QTEST_ENVIRONMENT")) {
217 for (const QByteArray &k : qgetenv(varName: "QTEST_ENVIRONMENT").split(sep: ' '))
218 result.insert(value: k);
219 }
220
221 return result;
222}
223
224static bool checkCondition(const QByteArray &condition)
225{
226 static const QSet<QByteArray> matchedConditions = activeConditions();
227 QList<QByteArray> conds = condition.split(sep: ' ');
228
229 for (QByteArray c : conds) {
230 bool result = c.startsWith(c: '!');
231 if (result)
232 c.remove(index: 0, len: 1);
233
234 result ^= matchedConditions.contains(value: c);
235 if (!result)
236 return false;
237 }
238 return true;
239}
240
241static bool ignoreAll = false;
242static std::set<QByteArray> *ignoredTests = nullptr;
243
244namespace QTestPrivate {
245
246void parseBlackList()
247{
248 QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
249 if (filename.isEmpty())
250 return;
251 QFile ignored(filename);
252 if (!ignored.open(flags: QIODevice::ReadOnly))
253 return;
254
255 QByteArray function;
256
257 while (!ignored.atEnd()) {
258 QByteArray line = ignored.readLine();
259 const int commentPosition = line.indexOf(c: '#');
260 if (commentPosition >= 0)
261 line.truncate(pos: commentPosition);
262 line = line.simplified();
263 if (line.isEmpty())
264 continue;
265 if (line.startsWith(c: '[')) {
266 function = line.mid(index: 1, len: line.size() - 2);
267 continue;
268 }
269 bool condition = checkCondition(condition: line);
270 if (condition) {
271 if (!function.size()) {
272 ignoreAll = true;
273 } else {
274 if (!ignoredTests)
275 ignoredTests = new std::set<QByteArray>;
276 ignoredTests->insert(x: function);
277 }
278 }
279 }
280}
281
282void checkBlackLists(const char *slot, const char *data, const char *global)
283{
284 bool ignore = ignoreAll;
285
286 if (!ignore && ignoredTests) {
287 QByteArray s = slot;
288 ignore = ignoredTests->find(x: s) != ignoredTests->end();
289 if (!ignore && data) {
290 s = (s + ':') + data;
291 ignore = ignoredTests->find(x: s) != ignoredTests->end();
292 }
293
294 if (!ignore && global) {
295 s = slot + ":"_ba + global;
296 ignore = ignoredTests->find(x: s) != ignoredTests->end();
297 if (!ignore && data) {
298 s = (s + ':') + data;
299 ignore = ignoredTests->find(x: s) != ignoredTests->end();
300 }
301 }
302 }
303
304 QTestResult::setBlacklistCurrentTest(ignore);
305}
306
307} // QTestPrivate
308
309QT_END_NAMESPACE
310

source code of qtbase/src/testlib/qtestblacklist.cpp