1/****************************************************************************
2**
3** Copyright (C) 2018 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29#include "abstracttestsuite.h"
30#include <QtTest/QtTest>
31#include <QtCore/qset.h>
32#include <QtCore/QSysInfo>
33#include <QtCore/qtextstream.h>
34#include <private/qmetaobjectbuilder_p.h>
35
36/*!
37 AbstractTestSuite provides a way of building QtTest test objects
38 dynamically. The use case is integration of JavaScript test suites
39 into QtTest autotests.
40
41 Subclasses add their tests functions with addTestFunction() in the
42 constructor, and must reimplement runTestFunction(). Additionally,
43 subclasses can reimplement initTestCase() and cleanupTestCase()
44 (but make sure to call the base implementation).
45
46 AbstractTestSuite uses configuration files for getting information
47 about skipped tests (skip.txt) and expected test failures
48 (expect_fail.txt). Subclasses must reimplement
49 createSkipConfigFile() and createExpectFailConfigFile() for
50 creating these files, and configData() for processing an entry of
51 such a file.
52
53 The config file format is as follows:
54 - Lines starting with '#' are skipped.
55 - Lines of the form [SYMBOL] means that the upcoming data
56 should only be processed if the given SYMBOL is defined on
57 this platform.
58 - Any other line is split on ' | ' and handed off to the client.
59
60 Subclasses must provide a default tests directory (where the
61 subclass expects to find the script files to run as tests), and a
62 default config file directory. Some environment variables can be
63 used to affect where AbstractTestSuite will look for files:
64
65 - QTSCRIPT_TEST_CONFIG_DIR: Overrides the default test config path.
66
67 - QTSCRIPT_TEST_CONFIG_SUFFIX: Is appended to "skip" and
68 "expect_fail" to create the test config name. This makes it easy to
69 maintain skip- and expect_fail-files corresponding to different
70 revisions of a test suite, and switch between them.
71
72 - QTSCRIPT_TEST_DIR: Overrides the default test dir.
73
74 AbstractTestSuite does _not_ define how the test dir itself is
75 processed or how tests are run; this is left up to the subclass.
76
77 If no config files are found, AbstractTestSuite will ask the
78 subclass to create a default skip file. Also, the
79 shouldGenerateExpectedFailures variable will be set to true. The
80 subclass should check for this when a test fails, and add an entry
81 to its set of expected failures. When all tests have been run,
82 AbstractTestSuite will ask the subclass to create the expect_fail
83 file based on the tests that failed. The next time the autotest is
84 run, the created config files will be used.
85
86 The reason for skipping a test is usually that it takes a very long
87 time to complete (or even hangs completely), or it crashes. It's
88 not possible for the test runner to know in advance which tests are
89 problematic, which is why the entries to the skip file are
90 typically added manually. When running tests for the first time, it
91 can be useful to run the autotest with the -v1 command line option,
92 so you can see the name of each test before it's run, and can add a
93 skip entry if appropriate.
94*/
95
96class TestConfigClientInterface;
97// For parsing information about skipped tests and expected failures.
98class TestConfigParser
99{
100public:
101 static void parse(const QString &path,
102 TestConfig::Mode mode,
103 TestConfigClientInterface *client);
104
105private:
106 static QString unescape(const QString &);
107 static bool isKnownSymbol(const QString &);
108 static bool isDefined(const QString &);
109
110 static QSet<QString> knownSymbols;
111 static QSet<QString> definedSymbols;
112};
113
114QSet<QString> TestConfigParser::knownSymbols;
115QSet<QString> TestConfigParser::definedSymbols;
116
117/**
118 Parses the config file at the given \a path in the given \a mode.
119 Handling of errors and data is delegated to the given \a client.
120*/
121void TestConfigParser::parse(const QString &path,
122 TestConfig::Mode mode,
123 TestConfigClientInterface *client)
124{
125 QFile file(path);
126 if (!file.open(flags: QIODevice::ReadOnly))
127 return;
128 QTextStream stream(&file);
129 int lineNumber = 0;
130 QString predicate;
131 const QString separator = QString::fromLatin1(str: " | ");
132 while (!stream.atEnd()) {
133 ++lineNumber;
134 QString line = stream.readLine();
135 if (line.isEmpty())
136 continue;
137 if (line.startsWith(c: '#')) // Comment
138 continue;
139 if (line.startsWith(c: '[')) { // Predicate
140 if (!line.endsWith(c: ']')) {
141 client->configError(path, message: "malformed predicate", lineNumber);
142 return;
143 }
144 QString symbol = line.mid(position: 1, n: line.size()-2);
145 if (isKnownSymbol(symbol)) {
146 predicate = symbol;
147 } else {
148 qWarning(msg: "symbol %s is not known -- add it to TestConfigParser!", qPrintable(symbol));
149 predicate = QString();
150 }
151 } else {
152 if (predicate.isEmpty() || isDefined(predicate)) {
153 QStringList parts = line.split(sep: separator, behavior: Qt::KeepEmptyParts);
154 for (int i = 0; i < parts.size(); ++i)
155 parts[i] = unescape(parts[i]);
156 client->configData(mode, parts);
157 }
158 }
159 }
160}
161
162QString TestConfigParser::unescape(const QString &str)
163{
164 return QString(str).replace(before: "\\n", after: "\n");
165}
166
167bool TestConfigParser::isKnownSymbol(const QString &symbol)
168{
169 if (knownSymbols.isEmpty()) {
170 knownSymbols
171 // If you add a symbol here, add a case for it in
172 // isDefined() as well.
173 << "Q_OS_LINUX"
174 << "Q_OS_SOLARIS"
175 << "Q_OS_WINCE"
176 << "Q_OS_SYMBIAN"
177 << "Q_OS_MAC"
178 << "Q_OS_WIN"
179 << "Q_CC_MSVC"
180 << "Q_CC_MSVC32"
181 << "Q_CC_MSVC64"
182 << "Q_CC_MINGW"
183 << "Q_CC_MINGW32"
184 << "Q_CC_MINGW64"
185 << "Q_CC_INTEL"
186 << "Q_CC_INTEL32"
187 << "Q_CC_INTEL64"
188 ;
189 }
190 return knownSymbols.contains(value: symbol);
191}
192
193bool TestConfigParser::isDefined(const QString &symbol)
194{
195 if (definedSymbols.isEmpty()) {
196 definedSymbols
197#ifdef Q_OS_LINUX
198 << "Q_OS_LINUX"
199#endif
200#ifdef Q_OS_SOLARIS
201 << "Q_OS_SOLARIS"
202#endif
203#ifdef Q_OS_WINCE
204 << "Q_OS_WINCE"
205#endif
206#ifdef Q_OS_SYMBIAN
207 << "Q_OS_SYMBIAN"
208#endif
209#ifdef Q_OS_MAC
210 << "Q_OS_MAC"
211#endif
212#ifdef Q_OS_WIN
213 << "Q_OS_WIN"
214#endif
215#ifdef Q_CC_MSVC
216 << "Q_CC_MSVC"
217 << (QStringLiteral("Q_CC_MSVC") + QString::number(QSysInfo::WordSize))
218#endif
219#ifdef Q_CC_MINGW
220 << "Q_CC_MINGW"
221 << (QStringLiteral("Q_CC_MINGW") + QString::number(QSysInfo::WordSize))
222#endif
223#ifdef Q_CC_INTEL
224 << "Q_CC_INTEL"
225 << (QStringLiteral("Q_CC_INTEL") + QString::number(QSysInfo::WordSize))
226#endif
227 ;
228 }
229 return definedSymbols.contains(value: symbol);
230}
231
232
233const QMetaObject *AbstractTestSuite::metaObject() const
234{
235 return dynamicMetaObject;
236}
237
238void *AbstractTestSuite::qt_metacast(const char *_clname)
239{
240 if (!_clname) return 0;
241 if (!strcmp(s1: _clname, s2: dynamicMetaObject->className()))
242 return static_cast<void*>(const_cast<AbstractTestSuite*>(this));
243 return QObject::qt_metacast(_clname);
244}
245
246void AbstractTestSuite::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a)
247{
248 Q_UNUSED(_a);
249 if (_c == QMetaObject::InvokeMetaMethod) {
250 AbstractTestSuite *_t = static_cast<AbstractTestSuite *>(_o);
251 switch (_id) {
252 case 0:
253 _t->initTestCase();
254 break;
255 case 1:
256 _t->cleanupTestCase();
257 break;
258 default:
259 // If another method is added above, this offset must be adjusted.
260 _t->runTestFunction(index: _id - 2);
261 }
262 }
263}
264
265int AbstractTestSuite::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
266{
267 _id = QObject::qt_metacall(_c, _id, _a);
268 if (_id < 0)
269 return _id;
270 if (_c == QMetaObject::InvokeMetaMethod) {
271 Q_ASSERT(dynamicMetaObject->cast(this));
272 int ownMethodCount = dynamicMetaObject->methodCount() - dynamicMetaObject->methodOffset();
273 if (_id < ownMethodCount)
274 qt_static_metacall(o: this, _c, _id, _a);
275 _id -= ownMethodCount;
276 }
277 return _id;
278}
279
280void AbstractTestSuite::addPrivateSlot(const QByteArray &signature)
281{
282 QMetaMethodBuilder slot = metaBuilder->addSlot(signature);
283 slot.setAccess(QMetaMethod::Private);
284}
285
286AbstractTestSuite::AbstractTestSuite(const QByteArray &className,
287 const QString &defaultTestsPath,
288 const QString &defaultConfigPath)
289 : shouldGenerateExpectedFailures(false),
290 dynamicMetaObject(0),
291 metaBuilder(new QMetaObjectBuilder)
292{
293 metaBuilder->setSuperClass(&QObject::staticMetaObject);
294 metaBuilder->setClassName(className);
295 metaBuilder->setStaticMetacallFunction(qt_static_metacall);
296
297 QString testConfigPath = qgetenv(varName: "QTSCRIPT_TEST_CONFIG_DIR");
298 if (testConfigPath.isEmpty())
299 testConfigPath = defaultConfigPath;
300 QString configSuffix = qgetenv(varName: "QTSCRIPT_TEST_CONFIG_SUFFIX");
301 skipConfigPath = QString::fromLatin1(str: "%0/skip%1.txt")
302 .arg(a: testConfigPath).arg(a: configSuffix);
303 expectFailConfigPath = QString::fromLatin1(str: "%0/expect_fail%1.txt")
304 .arg(a: testConfigPath).arg(a: configSuffix);
305
306 QString testsPath = qgetenv(varName: "QTSCRIPT_TEST_DIR");
307 if (testsPath.isEmpty())
308 testsPath = defaultTestsPath;
309 testsDir = QDir(testsPath);
310
311 addTestFunction("initTestCase");
312 addTestFunction("cleanupTestCase");
313
314 // Subclass constructors should add their custom test functions to
315 // the meta-object and call finalizeMetaObject().
316}
317
318AbstractTestSuite::~AbstractTestSuite()
319{
320 free(ptr: dynamicMetaObject);
321}
322
323void AbstractTestSuite::addTestFunction(const QString &name,
324 DataFunctionCreation dfc)
325{
326 if (dfc == CreateDataFunction) {
327 QString dataSignature = QString::fromLatin1(str: "%0_data()").arg(a: name);
328 addPrivateSlot(signature: dataSignature.toLatin1());
329 }
330 QString signature = QString::fromLatin1(str: "%0()").arg(a: name);
331 addPrivateSlot(signature: signature.toLatin1());
332}
333
334void AbstractTestSuite::finalizeMetaObject()
335{
336 dynamicMetaObject = metaBuilder->toMetaObject();
337}
338
339void AbstractTestSuite::initTestCase()
340{
341 if (!testsDir.exists()) {
342 QString message = QString::fromLatin1(str: "tests directory (%0) doesn't exist.")
343 .arg(a: testsDir.path());
344 QFAIL(qPrintable(message));
345 return;
346 }
347
348 if (QFileInfo(skipConfigPath).exists())
349 TestConfigParser::parse(path: skipConfigPath, mode: TestConfig::Skip, client: this);
350 else
351 createSkipConfigFile();
352
353 if (QFileInfo(expectFailConfigPath).exists())
354 TestConfigParser::parse(path: expectFailConfigPath, mode: TestConfig::ExpectFail, client: this);
355 else
356 shouldGenerateExpectedFailures = true;
357}
358
359void AbstractTestSuite::cleanupTestCase()
360{
361 if (shouldGenerateExpectedFailures)
362 createExpectFailConfigFile();
363}
364
365void AbstractTestSuite::configError(const QString &path, const QString &message, int lineNumber)
366{
367 QString output;
368 output.append(s: path);
369 if (lineNumber != -1)
370 output.append(s: ":").append(s: QString::number(lineNumber));
371 output.append(s: ": ").append(s: message);
372 QFAIL(qPrintable(output));
373}
374
375void AbstractTestSuite::createSkipConfigFile()
376{
377 QFile file(skipConfigPath);
378 if (!file.open(flags: QIODevice::WriteOnly))
379 return;
380 QWARN(qPrintable(QString::fromLatin1("creating %0").arg(skipConfigPath)));
381 QTextStream stream(&file);
382
383 writeSkipConfigFile(stream);
384
385 file.close();
386}
387
388void AbstractTestSuite::createExpectFailConfigFile()
389{
390 QFile file(expectFailConfigPath);
391 if (!file.open(flags: QFile::WriteOnly))
392 return;
393 QWARN(qPrintable(QString::fromLatin1("creating %0").arg(expectFailConfigPath)));
394 QTextStream stream(&file);
395
396 writeExpectFailConfigFile(stream);
397
398 file.close();
399}
400
401/*!
402 Convenience function for reading all contents of a file.
403 */
404QString AbstractTestSuite::readFile(const QString &filename)
405{
406 QFile file(filename);
407 if (!file.open(flags: QFile::ReadOnly))
408 return QString();
409 QTextStream stream(&file);
410 stream.setCodec("UTF-8");
411 return stream.readAll();
412}
413
414/*!
415 Escapes characters in the string \a str so it's suitable for writing
416 to a config file.
417 */
418QString AbstractTestSuite::escape(const QString &str)
419{
420 return QString(str).replace(before: "\n", after: "\\n");
421}
422

source code of qtscript/tests/auto/qscriptv8testsuite/abstracttestsuite.cpp