1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3#include "qtestblacklist_p.h"
4#include <QtTest/qtestcase.h>
5#include <QtCore/qbytearray.h>
6#include <QtCore/qfile.h>
7#include <QtCore/qset.h>
8#include <QtCore/qcoreapplication.h>
9#include <QtCore/qvariant.h>
10#include <QtCore/QSysInfo>
11#include <QtCore/QOperatingSystemVersion>
12
13#include <set>
14
15QT_BEGIN_NAMESPACE
16
17using namespace Qt::StringLiterals;
18
19/*
20 The BLACKLIST file format is a grouped listing of keywords.
21
22 Blank lines and everything after # is simply ignored. An initial #-line
23 referring to this documentation is kind to readers. Comments can also be used
24 to indicate the reasons for ignoring particular cases. Please scope comments
25 to keywords if possible, to avoid confusion when additional keywords or tests
26 cases are added later.
27
28 Each blacklist line is interpreted as a list of keywords in an AND-relationship.
29 To blacklist a test for multiple platforms (OR-relationship), use separate lines.
30
31 The key "ci" applies only when run by COIN. Other keys name platforms, operating systems,
32 distributions, tool-chains or architectures; a ! prefix reverses what it
33 checks. A version, joined to a key (at present, only for distributions and
34 for msvc) with a hyphen, limits the key to the specific version. A keyword
35 line matches if every key on it applies to the present run. Successive lines
36 are alternate conditions for ignoring a test.
37
38 Ungrouped lines at the beginning of a file apply to the whole testcase. A
39 group starts with a [square-bracketed] identification of a test function to
40 ignore. For data-driven tests, this identification can be narrowed by the
41 inclusion of global and local data row tags, separated from the function name
42 and each other by colons. If both global and function-specific data rows tags
43 are supplied, the global one comes first (as in the tag reported in test
44 output, albeit in parentheses after the function name). Even when a test does
45 have global and local data tags, you can omit either or both. (If a global
46 data row's name coincides with that of a local data row, some unintended
47 matches may result; try to keep your data-row tags distinct.)
48
49 Subsequent lines give conditions for ignoring this test. You need at least
50 one or the group has no effect.
51
52 # See qtbase/src/testlib/qtestblacklist.cpp for format
53 # Test doesn't work on QNX at all
54 qnx
55
56 [testFunction]
57 linux # QTBUG-12345
58 windows 64bit # QTBUG-12345
59
60 [testSlowly]
61 macos ci # Flaky in COIN on macOS, not reproducible by developers
62
63 [testfunction2:testData]
64 msvc-2010 # Needs basic C++11 support
65
66 [getFile:withProxy SSL:localhost]
67 android
68
69 QML test functions are identified using the following format:
70
71 <TestCase name>::<function name>:<data tag>
72
73 For example, to blacklist a QML test on RHEL 7.6:
74
75 [Button::test_display:TextOnly]
76 ci rhel-7.6 # QTBUG-12345
77
78 Keys are lower-case. Distribution name and version are supported if
79 QSysInfo's productType() and productVersion() return them.
80
81 Keys can be added via the space-separated QTEST_ENVIRONMENT
82 environment variable:
83
84 QTEST_ENVIRONMENT=ci ./tst_stuff
85
86 This can be used to "mock" a test environment. In the example above,
87 we add "ci" to the list of keys for the test environment, making it
88 possible to test BLACKLIST files that blacklist tests in a CI environment.
89
90 The other known keys are listed below:
91*/
92
93static QSet<QByteArray> keywords()
94{
95 // this list can be extended with new keywords as required
96 QSet<QByteArray> set = QSet<QByteArray>()
97 << "*"
98#ifdef Q_OS_LINUX
99 << "linux"
100#endif
101#ifdef Q_OS_MACOS
102 << "osx"
103 << "macos"
104#endif
105#if defined(Q_OS_WIN)
106 << "windows"
107#endif
108#ifdef Q_OS_IOS
109 << "ios"
110#endif
111#ifdef Q_OS_TVOS
112 << "tvos"
113#endif
114#ifdef Q_OS_WATCHOS
115 << "watchos"
116#endif
117#ifdef Q_OS_VISIONOS
118 << "visionos"
119#endif
120#ifdef Q_OS_ANDROID
121 << "android"
122#endif
123#ifdef Q_OS_QNX
124 << "qnx"
125#endif
126#ifdef Q_OS_VXWORKS
127 << "vxworks"
128#endif
129#ifdef Q_OS_WEBOS
130 << "webos"
131#endif
132
133#if QT_POINTER_SIZE == 8
134 << "64bit"
135#else
136 << "32bit"
137#endif
138
139#ifdef Q_CC_GNU
140 << "gcc"
141#endif
142#ifdef Q_CC_CLANG
143 << "clang"
144#endif
145#ifdef Q_CC_MSVC
146 << "msvc"
147# if _MSC_VER <= 1600
148 << "msvc-2010"
149# elif _MSC_VER <= 1700
150 << "msvc-2012"
151# elif _MSC_VER <= 1800
152 << "msvc-2013"
153# elif _MSC_VER <= 1900
154 << "msvc-2015"
155# elif _MSC_VER <= 1916
156 << "msvc-2017"
157# elif _MSC_VER <= 1929
158 << "msvc-2019"
159# else
160 << "msvc-2022"
161# endif
162#endif
163
164#ifdef Q_PROCESSOR_X86
165 << "x86"
166#endif
167#ifdef Q_PROCESSOR_ARM
168 << "arm"
169#endif
170
171#ifdef QT_BUILD_INTERNAL
172 << "developer-build"
173#endif
174 ;
175
176 QCoreApplication *app = QCoreApplication::instance();
177 if (app) {
178 const QVariant platformName = app->property(name: "platformName");
179 if (platformName.isValid())
180 set << platformName.toByteArray();
181 }
182
183 return set;
184}
185
186static QSet<QByteArray> activeConditions()
187{
188 QSet<QByteArray> result = keywords();
189
190 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
191 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
192 if (!distributionName.isEmpty()) {
193 if (result.find(value: distributionName) == result.end())
194 result.insert(value: distributionName);
195 // backwards compatibility with Qt 5
196 if (distributionName == "macos") {
197 if (result.find(value: distributionName) == result.end())
198 result.insert(value: "osx");
199 const auto version = QOperatingSystemVersion::current();
200 if (version.majorVersion() >= 11)
201 distributionRelease = QByteArray::number(version.majorVersion());
202 }
203 if (!distributionRelease.isEmpty()) {
204 QByteArray versioned = distributionName + "-" + distributionRelease;
205 if (result.find(value: versioned) == result.end())
206 result.insert(value: versioned);
207 if (distributionName == "macos") {
208 QByteArray versioned = "osx-" + distributionRelease;
209 if (result.find(value: versioned) == result.end())
210 result.insert(value: versioned);
211 }
212 }
213 }
214
215 if (qEnvironmentVariableIsSet(varName: "QTEST_ENVIRONMENT")) {
216 for (const QByteArray &k : qgetenv(varName: "QTEST_ENVIRONMENT").split(sep: ' '))
217 result.insert(value: k);
218 }
219
220 return result;
221}
222
223static bool checkCondition(const QByteArray &condition)
224{
225 static const QSet<QByteArray> matchedConditions = activeConditions();
226 QList<QByteArray> conds = condition.split(sep: ' ');
227
228 for (QByteArray c : conds) {
229 bool result = c.startsWith(c: '!');
230 if (result)
231 c.remove(index: 0, len: 1);
232
233 result ^= matchedConditions.contains(value: c);
234 if (!result)
235 return false;
236 }
237 return true;
238}
239
240static bool ignoreAll = false;
241static std::set<QByteArray> *ignoredTests = nullptr;
242
243namespace QTestPrivate {
244
245void parseBlackList()
246{
247 const QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
248 if (filename.isEmpty())
249 return;
250 QFile ignored(filename);
251 if (!ignored.open(flags: QIODevice::ReadOnly))
252 return;
253
254 QByteArray function;
255
256 while (!ignored.atEnd()) {
257 QByteArray line = ignored.readLine();
258 const int commentPosition = line.indexOf(ch: '#');
259 if (commentPosition >= 0)
260 line.truncate(pos: commentPosition);
261 line = line.simplified();
262 if (line.isEmpty())
263 continue;
264 if (line.startsWith(c: '[')) {
265 function = line.mid(index: 1, len: line.size() - 2);
266 continue;
267 }
268 const bool condition = checkCondition(condition: line);
269 if (condition) {
270 if (!function.size()) {
271 ignoreAll = true;
272 } else {
273 if (!ignoredTests)
274 ignoredTests = new std::set<QByteArray>;
275 ignoredTests->insert(x: function);
276 }
277 }
278 }
279}
280
281// Returns \c true if this test-case is blacklisted.
282bool checkBlackLists(const char *slot, const char *data, const char *global)
283{
284 bool ignore = ignoreAll;
285
286 if (!ignore && ignoredTests) {
287 QByteArray s = slot;
288 ignore = ignoredTests->find(x: s) != ignoredTests->end();
289 if (!ignore && data) {
290 s = (s + ':') + data;
291 ignore = ignoredTests->find(x: s) != ignoredTests->end();
292 }
293
294 if (!ignore && global) {
295 s = slot + ":"_ba + global;
296 ignore = ignoredTests->find(x: s) != ignoredTests->end();
297 if (!ignore && data) {
298 s = (s + ':') + data;
299 ignore = ignoredTests->find(x: s) != ignoredTests->end();
300 }
301 }
302 }
303
304 return ignore;
305}
306
307} // QTestPrivate
308
309QT_END_NAMESPACE
310

source code of qtbase/src/testlib/qtestblacklist.cpp