1/****************************************************************************
2**
3** Copyright (C) 2018 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29
30#include "abstracttestsuite.h"
31#include <QtTest/QtTest>
32
33#include <QtScript>
34
35struct TestRecord
36{
37 TestRecord() : lineNumber(-1) { }
38 TestRecord(const QString &desc,
39 bool pass,
40 const QString &act,
41 const QString &exp,
42 const QString &fn, int ln)
43 : description(desc), passed(pass),
44 actual(act), expected(exp),
45 fileName(fn), lineNumber(ln)
46 { }
47 TestRecord(const QString &skipReason, const QString &fn)
48 : description(skipReason), actual("QSKIP"),
49 fileName(fn), lineNumber(-1)
50 { }
51 QString description;
52 bool passed;
53 QString actual;
54 QString expected;
55 QString fileName;
56 int lineNumber;
57};
58
59Q_DECLARE_METATYPE(TestRecord)
60
61struct FailureItem
62{
63 enum Action {
64 ExpectFail,
65 Skip
66 };
67 FailureItem(Action act, const QRegExp &rx, const QString &desc, const QString &msg)
68 : action(act), pathRegExp(rx), description(desc), message(msg)
69 { }
70
71 Action action;
72 QRegExp pathRegExp;
73 QString description;
74 QString message;
75};
76
77class tst_QScriptJSTestSuite : public AbstractTestSuite
78{
79
80public:
81 tst_QScriptJSTestSuite();
82 virtual ~tst_QScriptJSTestSuite();
83
84protected:
85 virtual void configData(TestConfig::Mode mode, const QStringList &parts);
86 virtual void writeSkipConfigFile(QTextStream &);
87 virtual void writeExpectFailConfigFile(QTextStream &);
88 virtual void runTestFunction(int testIndex);
89
90private:
91 void addExpectedFailure(const QString &fileName, const QString &description, const QString &message);
92 void addExpectedFailure(const QRegExp &path, const QString &description, const QString &message);
93 void addSkip(const QString &fileName, const QString &description, const QString &message);
94 void addSkip(const QRegExp &path, const QString &description, const QString &message);
95 bool isExpectedFailure(const QString &fileName, const QString &description,
96 QString *message, FailureItem::Action *action) const;
97 void addFileExclusion(const QString &fileName, const QString &message);
98 void addFileExclusion(const QRegExp &rx, const QString &message);
99 bool isExcludedFile(const QString &fileName, QString *message) const;
100
101 QList<QString> subSuitePaths;
102 QList<FailureItem> expectedFailures;
103 QList<QPair<QRegExp, QString> > fileExclusions;
104};
105
106static QScriptValue qscript_void(QScriptContext *, QScriptEngine *eng)
107{
108 return eng->undefinedValue();
109}
110
111static QScriptValue qscript_quit(QScriptContext *ctx, QScriptEngine *)
112{
113 return ctx->throwError(text: "Test quit");
114}
115
116static QString optionsToString(int options)
117{
118 QSet<QString> set;
119 if (options & 1)
120 set.insert(value: "strict");
121 if (options & 2)
122 set.insert(value: "werror");
123 if (options & 4)
124 set.insert(value: "atline");
125 if (options & 8)
126 set.insert(value: "xml");
127 return QStringList(set.values()).join(sep: ",");
128}
129
130static QScriptValue qscript_options(QScriptContext *ctx, QScriptEngine *)
131{
132 static QHash<QString, int> stringToFlagHash;
133 if (stringToFlagHash.isEmpty()) {
134 stringToFlagHash["strict"] = 1;
135 stringToFlagHash["werror"] = 2;
136 stringToFlagHash["atline"] = 4;
137 stringToFlagHash["xml"] = 8;
138 }
139 QScriptValue callee = ctx->callee();
140 int opts = callee.data().toInt32();
141 QString result = optionsToString(options: opts);
142 for (int i = 0; i < ctx->argumentCount(); ++i)
143 opts |= stringToFlagHash.value(akey: ctx->argument(index: 0).toString());
144 callee.setData(opts);
145 return result;
146}
147
148static QScriptValue qscript_TestCase(QScriptContext *ctx, QScriptEngine *eng)
149{
150 QScriptValue origTestCaseCtor = ctx->callee().data();
151 QScriptValue kase = ctx->thisObject();
152 QScriptValue ret = origTestCaseCtor.call(thisObject: kase, arguments: ctx->argumentsObject());
153 QScriptContextInfo info(ctx->parentContext());
154 kase.setProperty(name: "__lineNumber__", value: QScriptValue(eng, info.lineNumber()));
155 return ret;
156}
157
158void tst_QScriptJSTestSuite::runTestFunction(int testIndex)
159{
160 if (!(testIndex & 1)) {
161 // data
162 QTest::addColumn<TestRecord>(name: "record");
163 bool hasData = false;
164
165 QString testsShellPath = testsDir.absoluteFilePath(fileName: "shell.js");
166 QString testsShellContents = readFile(testsShellPath);
167
168 QDir subSuiteDir(subSuitePaths.at(i: testIndex / 2));
169 QString subSuiteShellPath = subSuiteDir.absoluteFilePath(fileName: "shell.js");
170 QString subSuiteShellContents = readFile(subSuiteShellPath);
171
172 QDir testSuiteDir(subSuiteDir);
173 testSuiteDir.cdUp();
174 QString suiteJsrefPath = testSuiteDir.absoluteFilePath(fileName: "jsref.js");
175 QString suiteJsrefContents = readFile(suiteJsrefPath);
176 QString suiteShellPath = testSuiteDir.absoluteFilePath(fileName: "shell.js");
177 QString suiteShellContents = readFile(suiteShellPath);
178
179 const QFileInfoList testFileInfos = subSuiteDir.entryInfoList(nameFilters: QStringList() << "*.js", filters: QDir::Files);
180 for (const QFileInfo &tfi : testFileInfos) {
181 if ((tfi.fileName() == "shell.js") || (tfi.fileName() == "browser.js"))
182 continue;
183
184 QString abspath = tfi.absoluteFilePath();
185 QString relpath = testsDir.relativeFilePath(fileName: abspath);
186 QString excludeMessage;
187 if (isExcludedFile(fileName: relpath, message: &excludeMessage)) {
188 QTest::newRow(dataTag: relpath.toLatin1()) << TestRecord(excludeMessage, relpath);
189 continue;
190 }
191
192 QScriptEngine eng;
193 QScriptValue global = eng.globalObject();
194 global.setProperty(name: "print", value: eng.newFunction(signature: qscript_void));
195 global.setProperty(name: "quit", value: eng.newFunction(signature: qscript_quit));
196 global.setProperty(name: "options", value: eng.newFunction(signature: qscript_options));
197
198 eng.evaluate(program: testsShellContents, fileName: testsShellPath);
199 if (eng.hasUncaughtException()) {
200 QStringList bt = eng.uncaughtExceptionBacktrace();
201 QString err = eng.uncaughtException().toString();
202 qWarning(msg: "%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
203 break;
204 }
205
206 eng.evaluate(program: suiteJsrefContents, fileName: suiteJsrefPath);
207 if (eng.hasUncaughtException()) {
208 QStringList bt = eng.uncaughtExceptionBacktrace();
209 QString err = eng.uncaughtException().toString();
210 qWarning(msg: "%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
211 break;
212 }
213
214 eng.evaluate(program: suiteShellContents, fileName: suiteShellPath);
215 if (eng.hasUncaughtException()) {
216 QStringList bt = eng.uncaughtExceptionBacktrace();
217 QString err = eng.uncaughtException().toString();
218 qWarning(msg: "%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
219 break;
220 }
221
222 eng.evaluate(program: subSuiteShellContents, fileName: subSuiteShellPath);
223 if (eng.hasUncaughtException()) {
224 QStringList bt = eng.uncaughtExceptionBacktrace();
225 QString err = eng.uncaughtException().toString();
226 qWarning(msg: "%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
227 break;
228 }
229
230 QScriptValue origTestCaseCtor = global.property(name: "TestCase");
231 QScriptValue myTestCaseCtor = eng.newFunction(signature: qscript_TestCase);
232 myTestCaseCtor.setData(origTestCaseCtor);
233 global.setProperty(name: "TestCase", value: myTestCaseCtor);
234
235 global.setProperty(name: "gTestfile", value: tfi.fileName());
236 global.setProperty(name: "gTestsuite", value: testSuiteDir.dirName());
237 global.setProperty(name: "gTestsubsuite", value: subSuiteDir.dirName());
238 QString testFileContents = readFile(abspath);
239// qDebug() << relpath;
240 eng.evaluate(program: testFileContents, fileName: abspath);
241 if (eng.hasUncaughtException() && !relpath.endsWith(s: "-n.js")) {
242 QStringList bt = eng.uncaughtExceptionBacktrace();
243 QString err = eng.uncaughtException().toString();
244 qWarning(msg: "%s\n%s\n", qPrintable(err), qPrintable(bt.join("\n")));
245 continue;
246 }
247
248 QScriptValue testcases = global.property(name: "testcases");
249 if (!testcases.isArray())
250 testcases = global.property(name: "gTestcases");
251 int count = testcases.property(name: "length").toInt32();
252 if (count == 0)
253 continue;
254
255 hasData = true;
256 QString title = global.property(name: "TITLE").toString();
257 for (int i = 0; i < count; ++i) {
258 QScriptValue kase = testcases.property(arrayIndex: i);
259 QString description = kase.property(name: "description").toString();
260 QScriptValue expect = kase.property(name: "expect");
261 QScriptValue actual = kase.property(name: "actual");
262 bool passed = kase.property(name: "passed").toBoolean();
263 int lineNumber = kase.property(name: "__lineNumber__").toInt32();
264
265 TestRecord rec(description, passed,
266 actual.toString(), expect.toString(),
267 relpath, lineNumber);
268
269 QTest::newRow(dataTag: description.toUtf8()) << rec;
270 }
271 }
272 if (!hasData)
273 QTest::newRow(dataTag: "") << TestRecord(); // dummy
274 } else {
275 QFETCH(TestRecord, record);
276 if ((record.lineNumber == -1) && (record.actual == "QSKIP")) {
277 QTest::qSkip(message: record.description.toLatin1(), file: record.fileName.toLatin1(), line: -1);
278 } else {
279 QString msg;
280 FailureItem::Action failAct;
281 bool expectFail = isExpectedFailure(fileName: record.fileName, description: record.description, message: &msg, action: &failAct);
282 if (expectFail) {
283 switch (failAct) {
284 case FailureItem::ExpectFail:
285 QTest::qExpectFail(dataIndex: "", comment: msg.toLatin1(),
286 mode: QTest::Continue, file: record.fileName.toLatin1(),
287 line: record.lineNumber);
288 break;
289 case FailureItem::Skip:
290 QTest::qSkip(message: msg.toLatin1(), file: record.fileName.toLatin1(), line: record.lineNumber);
291 break;
292 }
293 }
294 if (!expectFail || (failAct == FailureItem::ExpectFail)) {
295 if (!record.passed) {
296 if (!expectFail && shouldGenerateExpectedFailures) {
297 addExpectedFailure(fileName: record.fileName,
298 description: record.description,
299 message: QString());
300 }
301 QTest::qCompare(t1: record.actual, t2: record.expected, actual: "actual", expected: "expect",
302 file: record.fileName.toLatin1(), line: record.lineNumber);
303 } else {
304 QTest::qCompare(t1: record.actual, t2: record.actual, actual: "actual", expected: "expect",
305 file: record.fileName.toLatin1(), line: record.lineNumber);
306 }
307 }
308 }
309 }
310}
311
312tst_QScriptJSTestSuite::tst_QScriptJSTestSuite()
313 : AbstractTestSuite("tst_QScriptJsTestSuite",
314 QFINDTESTDATA("tests"),
315 ":/")
316{
317// don't execute any tests on slow machines
318#if !defined(Q_OS_IRIX)
319 // do all the test suites
320 const QFileInfoList testSuiteDirInfos = testsDir.entryInfoList(filters: QDir::AllDirs | QDir::NoDotAndDotDot);
321 for (const QFileInfo &tsdi : testSuiteDirInfos) {
322 QDir testSuiteDir(tsdi.absoluteFilePath());
323 // do all the dirs in the test suite
324 const QFileInfoList subSuiteDirInfos = testSuiteDir.entryInfoList(filters: QDir::AllDirs | QDir::NoDotAndDotDot);
325 for (const QFileInfo &ssdi : subSuiteDirInfos) {
326 subSuitePaths.append(t: ssdi.absoluteFilePath());
327 QString function = QString::fromLatin1(str: "%0/%1")
328 .arg(a: testSuiteDir.dirName()).arg(a: ssdi.fileName());
329 addTestFunction(function, CreateDataFunction);
330 }
331 }
332#endif
333
334 finalizeMetaObject();
335}
336
337tst_QScriptJSTestSuite::~tst_QScriptJSTestSuite()
338{
339}
340
341void tst_QScriptJSTestSuite::configData(TestConfig::Mode mode, const QStringList &parts)
342{
343 switch (mode) {
344 case TestConfig::Skip:
345 addFileExclusion(fileName: parts.at(i: 0), message: parts.value(i: 1));
346 break;
347
348 case TestConfig::ExpectFail:
349 addExpectedFailure(fileName: parts.at(i: 0), description: parts.value(i: 1), message: parts.value(i: 2));
350 break;
351 }
352}
353
354void tst_QScriptJSTestSuite::writeSkipConfigFile(QTextStream &stream)
355{
356 stream << QString::fromLatin1(str: "# testcase | message") << endl;
357}
358
359void tst_QScriptJSTestSuite::writeExpectFailConfigFile(QTextStream &stream)
360{
361 stream << QString::fromLatin1(str: "# testcase | description | message") << endl;
362 for (int i = 0; i < expectedFailures.size(); ++i) {
363 const FailureItem &fail = expectedFailures.at(i);
364 if (fail.pathRegExp.pattern().isEmpty())
365 continue;
366 stream << QString::fromLatin1(str: "%0 | %1")
367 .arg(a: fail.pathRegExp.pattern())
368 .arg(a: escape(fail.description));
369 if (!fail.message.isEmpty())
370 stream << QString::fromLatin1(str: " | %0").arg(a: escape(fail.message));
371 stream << endl;
372 }
373}
374
375void tst_QScriptJSTestSuite::addExpectedFailure(const QRegExp &path, const QString &description, const QString &message)
376{
377 expectedFailures.append(t: FailureItem(FailureItem::ExpectFail, path, description, message));
378}
379
380void tst_QScriptJSTestSuite::addExpectedFailure(const QString &fileName, const QString &description, const QString &message)
381{
382 expectedFailures.append(t: FailureItem(FailureItem::ExpectFail, QRegExp(fileName), description, message));
383}
384
385void tst_QScriptJSTestSuite::addSkip(const QRegExp &path, const QString &description, const QString &message)
386{
387 expectedFailures.append(t: FailureItem(FailureItem::Skip, path, description, message));
388}
389
390void tst_QScriptJSTestSuite::addSkip(const QString &fileName, const QString &description, const QString &message)
391{
392 expectedFailures.append(t: FailureItem(FailureItem::Skip, QRegExp(fileName), description, message));
393}
394
395bool tst_QScriptJSTestSuite::isExpectedFailure(const QString &fileName, const QString &description,
396 QString *message, FailureItem::Action *action) const
397{
398 for (int i = 0; i < expectedFailures.size(); ++i) {
399 QRegExp pathRegExp = expectedFailures.at(i).pathRegExp;
400 if (pathRegExp.indexIn(str: fileName) != -1) {
401 if (description == expectedFailures.at(i).description) {
402 if (message)
403 *message = expectedFailures.at(i).message;
404 if (action)
405 *action = expectedFailures.at(i).action;
406 return true;
407 }
408 }
409 }
410 return false;
411}
412
413void tst_QScriptJSTestSuite::addFileExclusion(const QString &fileName, const QString &message)
414{
415 fileExclusions.append(t: qMakePair(x: QRegExp(fileName), y: message));
416}
417
418void tst_QScriptJSTestSuite::addFileExclusion(const QRegExp &rx, const QString &message)
419{
420 fileExclusions.append(t: qMakePair(x: rx, y: message));
421}
422
423bool tst_QScriptJSTestSuite::isExcludedFile(const QString &fileName, QString *message) const
424{
425 for (int i = 0; i < fileExclusions.size(); ++i) {
426 QRegExp copy = fileExclusions.at(i).first;
427 if (copy.indexIn(str: fileName) != -1) {
428 if (message)
429 *message = fileExclusions.at(i).second;
430 return true;
431 }
432 }
433 return false;
434}
435
436QTEST_MAIN(tst_QScriptJSTestSuite)
437

source code of qtscript/tests/auto/qscriptjstestsuite/tst_qscriptjstestsuite.cpp