1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39#include "qtestblacklist_p.h"
40#include "qtestresult_p.h"
41
42#include <QtTest/qtestcase.h>
43#include <QtCore/qbytearray.h>
44#include <QtCore/qfile.h>
45#include <QtCore/qset.h>
46#include <QtCore/qcoreapplication.h>
47#include <QtCore/qvariant.h>
48#include <QtCore/QSysInfo>
49#include <QtCore/QOperatingSystemVersion>
50
51#include <set>
52
53QT_BEGIN_NAMESPACE
54
55/*
56 The BLACKLIST file format is a grouped listing of keywords.
57
58 Blank lines and everything after # is simply ignored. An initial #-line
59 referring to this documentation is kind to readers. Comments can also be used
60 to indicate the reasons for ignoring particular cases.
61
62 The key "ci" applies only when run by COIN. Other keys name platforms,
63 operating systems, distributions, tool-chains or architectures; a ! prefix
64 reverses what it checks. A version, joined to a key (at present, only for
65 distributions and for msvc) with a hyphen, limits the key to the specific
66 version. A keyword line matches if every key on it applies to the present
67 run. Successive lines are alternate conditions for ignoring a test.
68
69 Ungrouped lines at the beginning of a file apply to the whole testcase.
70 A group starts with a [square-bracketed] identification of a test function,
71 optionally with (after a colon, the name of) a specific data set, to ignore.
72 Subsequent lines give conditions for ignoring this test.
73
74 # See qtbase/src/testlib/qtestblacklist.cpp for format
75 # Test doesn't work on QNX at all
76 qnx
77
78 # QTBUG-12345
79 [testFunction]
80 linux
81 windows 64bit
82
83 # Flaky in COIN on macOS, not reproducible by developers
84 [testSlowly]
85 ci osx
86
87 # Needs basic C++11 support
88 [testfunction2:testData]
89 msvc-2010
90
91 QML test functions are identified using the following format:
92
93 <TestCase name>::<function name>:<data tag>
94
95 For example, to blacklist a QML test on RHEL 7.6:
96
97 # QTBUG-12345
98 [Button::test_display:TextOnly]
99 ci rhel-7.6
100
101 Keys are lower-case. Distribution name and version are supported if
102 QSysInfo's productType() and productVersion() return them.
103
104 Keys can be added via the space-separated QTEST_ENVIRONMENT
105 environment variable:
106
107 QTEST_ENVIRONMENT=ci ./tst_stuff
108
109 This can be used to "mock" a test environment. In the example above,
110 we add "ci" to the list of keys for the test environment, making it
111 possible to test BLACKLIST files that blacklist tests in a CI environment.
112
113 The other known keys are listed below:
114*/
115
116static QSet<QByteArray> keywords()
117{
118 // this list can be extended with new keywords as required
119 QSet<QByteArray> set = QSet<QByteArray>()
120 << "*"
121#ifdef Q_OS_LINUX
122 << "linux"
123#endif
124#ifdef Q_OS_MACOS
125 << "osx"
126 << "macos"
127#endif
128#if defined(Q_OS_WIN) && !defined(Q_OS_WINRT)
129 << "windows"
130#endif
131#ifdef Q_OS_IOS
132 << "ios"
133#endif
134#ifdef Q_OS_TVOS
135 << "tvos"
136#endif
137#ifdef Q_OS_WATCHOS
138 << "watchos"
139#endif
140#ifdef Q_OS_ANDROID
141 << "android"
142#endif
143#ifdef Q_OS_QNX
144 << "qnx"
145#endif
146#ifdef Q_OS_WINRT
147 << "winrt"
148#endif
149
150#if QT_POINTER_SIZE == 8
151 << "64bit"
152#else
153 << "32bit"
154#endif
155
156#ifdef Q_CC_GNU
157 << "gcc"
158#endif
159#ifdef Q_CC_CLANG
160 << "clang"
161#endif
162#ifdef Q_CC_MSVC
163 << "msvc"
164# if _MSC_VER <= 1600
165 << "msvc-2010"
166# elif _MSC_VER <= 1700
167 << "msvc-2012"
168# elif _MSC_VER <= 1800
169 << "msvc-2013"
170# elif _MSC_VER <= 1900
171 << "msvc-2015"
172# elif _MSC_VER <= 1916
173 << "msvc-2017"
174# else
175 << "msvc-2019"
176# endif
177#endif
178
179#ifdef Q_PROCESSOR_X86
180 << "x86"
181#endif
182#ifdef Q_PROCESSOR_ARM
183 << "arm"
184#endif
185
186#ifdef QT_BUILD_INTERNAL
187 << "developer-build"
188#endif
189 ;
190
191#if QT_CONFIG(properties)
192 QCoreApplication *app = QCoreApplication::instance();
193 if (app) {
194 const QVariant platformName = app->property(name: "platformName");
195 if (platformName.isValid())
196 set << platformName.toByteArray();
197 }
198#endif
199
200 return set;
201}
202
203static QSet<QByteArray> activeConditions()
204{
205 QSet<QByteArray> result = keywords();
206
207 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
208 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
209 if (!distributionName.isEmpty()) {
210 if (result.find(value: distributionName) == result.end())
211 result.insert(value: distributionName);
212 if (distributionName == "macos" || distributionName == "osx") {
213 const auto version = QOperatingSystemVersion::current();
214 if (version.majorVersion() >= 11)
215 distributionRelease = QByteArray::number(version.majorVersion());
216 }
217 if (!distributionRelease.isEmpty()) {
218 QByteArray versioned = distributionName + "-" + distributionRelease;
219 if (result.find(value: versioned) == result.end())
220 result.insert(value: versioned);
221 }
222 }
223
224 if (qEnvironmentVariableIsSet(varName: "QTEST_ENVIRONMENT")) {
225 for (const QByteArray &k : qgetenv(varName: "QTEST_ENVIRONMENT").split(sep: ' '))
226 result.insert(value: k);
227 }
228
229 return result;
230}
231
232static bool checkCondition(const QByteArray &condition)
233{
234 static const QSet<QByteArray> matchedConditions = activeConditions();
235 QList<QByteArray> conds = condition.split(sep: ' ');
236
237 for (QByteArray c : conds) {
238 bool result = c.startsWith(c: '!');
239 if (result)
240 c.remove(index: 0, len: 1);
241
242 result ^= matchedConditions.contains(value: c);
243 if (!result)
244 return false;
245 }
246 return true;
247}
248
249static bool ignoreAll = false;
250static std::set<QByteArray> *ignoredTests = nullptr;
251
252namespace QTestPrivate {
253
254void parseBlackList()
255{
256 QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
257 if (filename.isEmpty())
258 return;
259 QFile ignored(filename);
260 if (!ignored.open(flags: QIODevice::ReadOnly))
261 return;
262
263 QByteArray function;
264
265 while (!ignored.atEnd()) {
266 QByteArray line = ignored.readLine();
267 const int commentPosition = line.indexOf(c: '#');
268 if (commentPosition >= 0)
269 line.truncate(pos: commentPosition);
270 line = line.simplified();
271 if (line.isEmpty())
272 continue;
273 if (line.startsWith(c: '[')) {
274 function = line.mid(index: 1, len: line.length() - 2);
275 continue;
276 }
277 bool condition = checkCondition(condition: line);
278 if (condition) {
279 if (!function.size()) {
280 ignoreAll = true;
281 } else {
282 if (!ignoredTests)
283 ignoredTests = new std::set<QByteArray>;
284 ignoredTests->insert(x: function);
285 }
286 }
287 }
288}
289
290void checkBlackLists(const char *slot, const char *data)
291{
292 bool ignore = ignoreAll;
293
294 if (!ignore && ignoredTests) {
295 QByteArray s = slot;
296 ignore = (ignoredTests->find(x: s) != ignoredTests->end());
297 if (!ignore && data) {
298 s += ':';
299 s += data;
300 ignore = (ignoredTests->find(x: s) != ignoredTests->end());
301 }
302 }
303
304 QTestResult::setBlacklistCurrentTest(ignore);
305}
306
307} // QTestPrivate
308
309QT_END_NAMESPACE
310

source code of qtbase/src/testlib/qtestblacklist.cpp