1/****************************************************************************
2**
3** Copyright (C) 2020 The Qt Company Ltd.
4** Copyright (C) 2016 Intel Corporation.
5** Contact: https://www.qt.io/licensing/
6**
7** This file is part of the test suite of the Qt Toolkit.
8**
9** $QT_BEGIN_LICENSE:GPL-EXCEPT$
10** Commercial License Usage
11** Licensees holding valid commercial Qt licenses may use this file in
12** accordance with the commercial license agreement provided with the
13** Software or, alternatively, in accordance with the terms contained in
14** a written agreement between you and The Qt Company. For licensing terms
15** and conditions see https://www.qt.io/terms-conditions. For further
16** information use the contact form at https://www.qt.io/contact-us.
17**
18** GNU General Public License Usage
19** Alternatively, this file may be used under the terms of the GNU
20** General Public License version 3 as published by the Free Software
21** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
22** included in the packaging of this file. Please review the following
23** information to ensure the GNU General Public License requirements will
24** be met: https://www.gnu.org/licenses/gpl-3.0.html.
25**
26** $QT_END_LICENSE$
27**
28****************************************************************************/
29
30#include <QtCore/QCoreApplication>
31
32#if QT_CONFIG(temporaryfile) && QT_CONFIG(process)
33# define USE_DIFF
34#endif
35#include <QtCore/QXmlStreamReader>
36#include <QtCore/QFileInfo>
37#include <QtCore/QDir>
38#include <QtCore/QTemporaryDir>
39#ifdef USE_DIFF
40# include <QtCore/QTemporaryFile>
41# include <QtCore/QStandardPaths>
42#endif
43#include <QtTest/QtTest>
44
45#include <private/cycle_p.h>
46
47#include "emulationdetector.h"
48
49struct LoggerSet;
50
51class tst_Selftests: public QObject
52{
53 Q_OBJECT
54public:
55 tst_Selftests();
56
57private slots:
58 void initTestCase();
59 void runSubTest_data();
60 void runSubTest();
61 void cleanup();
62
63private:
64 void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
65 bool compareOutput(const QString &logger, const QString &subdir,
66 const QByteArray &rawOutput, const QByteArrayList &actual,
67 const QByteArrayList &expected,
68 QString *errorMessage) const;
69 bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
70 const QString &actualLine, const QString &expLine,
71 QString *errorMessage) const;
72 bool checkXml(const QString &logger, QByteArray rawOutput,
73 QString *errorMessage) const;
74
75 QString logName(const QString &logger) const;
76 QList<LoggerSet> allLoggerSets() const;
77
78 QTemporaryDir tempDir;
79};
80
81struct BenchmarkResult
82{
83 qint64 total;
84 qint64 iterations;
85 QString unit;
86
87 inline QString toString() const
88 { return QString("total:%1, unit:%2, iterations:%3").arg(a: total).arg(a: unit).arg(a: iterations); }
89
90 static BenchmarkResult parse(QString const&, QString*);
91};
92
93static QString msgMismatch(const QString &actual, const QString &expected)
94{
95 return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
96 + expected + QLatin1Char('\'');
97}
98
99static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
100 QString *errorMessage)
101{
102 // First make sure the iterations and unit match.
103 if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
104 // Nope - compare whole string for best failure message
105 *errorMessage = msgMismatch(actual: r1.toString(), expected: r2.toString());
106 return false;
107 }
108
109 // Now check the value. Some variance is allowed, and how much depends on
110 // the measured unit.
111 qreal variance = 0.;
112 if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
113 variance = 0.1;
114 else if (r1.unit == QLatin1String("instruction reads"))
115 variance = 0.001;
116 else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
117 variance = 0.001;
118
119 if (variance == 0.) {
120 // No variance allowed - compare whole string
121 const QString r1S = r1.toString();
122 const QString r2S = r2.toString();
123 if (r1S != r2S) {
124 *errorMessage = msgMismatch(actual: r1S, expected: r2S);
125 return false;
126 }
127 return true;
128 }
129
130 if (qAbs(t: qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
131 // Whoops, didn't match. Compare the whole string for the most useful failure message.
132 *errorMessage = msgMismatch(actual: r1.toString(), expected: r2.toString());
133 return false;
134 }
135 return true;
136}
137
138// Split the passed block of text into an array of lines, replacing any
139// filenames and line numbers with generic markers to avoid failing the test
140// due to compiler-specific behaviour.
141static QList<QByteArray> splitLines(QByteArray ba)
142{
143 ba.replace(before: '\r', c: "");
144 QList<QByteArray> out = ba.split(sep: '\n');
145
146 // Replace any ` file="..."' or ` line="..."' in XML with a generic location.
147 static const char *markers[][2] = {
148 { " file=\"", " file=\"__FILE__\"" },
149 { " line=\"", " line=\"__LINE__\"" }
150 };
151 static const int markerCount = sizeof markers / sizeof markers[0];
152
153 for (int i = 0; i < out.size(); ++i) {
154 QByteArray& line = out[i];
155 for (int j = 0; j < markerCount; ++j) {
156 int index = line.indexOf(c: markers[j][0]);
157 if (index == -1) {
158 continue;
159 }
160 const int end = line.indexOf(c: '"', from: index + int(strlen(s: markers[j][0])));
161 if (end == -1) {
162 continue;
163 }
164 line.replace(index, len: end-index + 1, s: markers[j][1]);
165 }
166 }
167
168 return out;
169}
170
171// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'.
172static inline QString logFormat(const QString &logger)
173{
174 return (logger.startsWith(s: "stdout") ? logger.mid(position: 7) : logger);
175}
176
177// Return the log file name, or an empty string if the log goes to stdout.
178QString tst_Selftests::logName(const QString &logger) const
179{
180 return (logger.startsWith(s: "stdout") ? "" : QString(tempDir.path() + "/test_output." + logger));
181}
182
183static QString expectedFileNameFromTest(const QString &subdir, const QString &logger)
184{
185 return QStringLiteral("expected_") + subdir + QLatin1Char('.') + logFormat(logger);
186}
187
188// Load the expected test output for the nominated test (subdir) and logger
189// as an array of lines. If there is no expected output file, return an
190// empty array.
191static QList<QByteArray> expectedResult(const QString &fileName)
192{
193 QFile file(QStringLiteral(":/") + fileName);
194 if (!file.open(flags: QIODevice::ReadOnly))
195 return QList<QByteArray>();
196 return splitLines(ba: file.readAll());
197}
198
199// Helpers for running the 'diff' tool in case comparison fails
200#ifdef USE_DIFF
201static inline void writeLines(QIODevice &d, const QByteArrayList &lines)
202{
203 for (const QByteArray &l : lines) {
204 d.write(data: l);
205 d.write(data: "\n");
206 }
207}
208#endif // USE_DIFF
209
210static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual)
211{
212 QByteArray result;
213#ifdef USE_DIFF
214# ifndef Q_OS_WIN
215 const QString diff = QStandardPaths::findExecutable(executableName: "diff");
216# else
217 const QString diff = QStandardPaths::findExecutable("diff.exe");
218# endif
219 if (diff.isEmpty())
220 return result;
221 QTemporaryFile expectedFile;
222 if (!expectedFile.open())
223 return result;
224 writeLines(d&: expectedFile, lines: expected);
225 expectedFile.close();
226 QTemporaryFile actualFile;
227 if (!actualFile.open())
228 return result;
229 writeLines(d&: actualFile, lines: actual);
230 actualFile.close();
231 QProcess diffProcess;
232 diffProcess.start(program: diff, arguments: {QLatin1String("-u"), expectedFile.fileName(), actualFile.fileName()});
233 if (!diffProcess.waitForStarted())
234 return result;
235 if (diffProcess.waitForFinished())
236 result = diffProcess.readAllStandardOutput();
237 else
238 diffProcess.kill();
239#endif // USE_DIFF
240 return result;
241}
242
243// Each test is run with a set of one or more test output loggers.
244// This struct holds information about one such test.
245struct LoggerSet
246{
247 LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments)
248 : name(_name), loggers(_loggers), arguments(_arguments)
249 { }
250
251 QString name;
252 QStringList loggers;
253 QStringList arguments;
254};
255
256// This function returns a list of all sets of loggers to be used for
257// running each subtest.
258QList<LoggerSet> tst_Selftests::allLoggerSets() const
259{
260 // Note that in order to test XML output to standard output, the subtests
261 // must not send output directly to stdout, bypassing Qt's output mechanisms
262 // (e.g. via printf), otherwise the output may not be well-formed XML.
263 return QList<LoggerSet>()
264 // Test with old-style options for a single logger
265 << LoggerSet("old stdout txt",
266 QStringList() << "stdout txt",
267 QStringList()
268 )
269 << LoggerSet("old txt",
270 QStringList() << "txt",
271 QStringList() << "-o" << logName(logger: "txt")
272 )
273 << LoggerSet("old stdout xml",
274 QStringList() << "stdout xml",
275 QStringList() << "-xml"
276 )
277 << LoggerSet("old xml",
278 QStringList() << "xml",
279 QStringList() << "-xml" << "-o" << logName(logger: "xml")
280 )
281 << LoggerSet("old stdout junitxml",
282 QStringList() << "stdout junitxml",
283 QStringList() << "-junitxml"
284 )
285 << LoggerSet("old junitxml",
286 QStringList() << "junitxml",
287 QStringList() << "-junitxml" << "-o" << logName(logger: "junitxml")
288 )
289 << LoggerSet("old xunitxml compatibility",
290 QStringList() << "junitxml",
291 QStringList() << "-xunitxml" << "-o" << logName(logger: "junitxml")
292 )
293 << LoggerSet("old stdout lightxml",
294 QStringList() << "stdout lightxml",
295 QStringList() << "-lightxml"
296 )
297 << LoggerSet("old lightxml",
298 QStringList() << "lightxml",
299 QStringList() << "-lightxml" << "-o" << logName(logger: "lightxml")
300 )
301 << LoggerSet("old stdout csv", // benchmarks only
302 QStringList() << "stdout csv",
303 QStringList() << "-csv")
304 << LoggerSet("old csv", // benchmarks only
305 QStringList() << "csv",
306 QStringList() << "-csv" << "-o" << logName(logger: "csv"))
307 << LoggerSet("old stdout teamcity",
308 QStringList() << "stdout teamcity",
309 QStringList() << "-teamcity"
310 )
311 << LoggerSet("old teamcity",
312 QStringList() << "teamcity",
313 QStringList() << "-teamcity" << "-o" << logName(logger: "teamcity")
314 )
315 << LoggerSet("old stdout tap",
316 QStringList() << "stdout tap",
317 QStringList() << "-tap"
318 )
319 << LoggerSet("old tap",
320 QStringList() << "tap",
321 QStringList() << "-tap" << "-o" << logName(logger: "tap")
322 )
323 // Test with new-style options for a single logger
324 << LoggerSet("new stdout txt",
325 QStringList() << "stdout txt",
326 QStringList() << "-o" << "-,txt"
327 )
328 << LoggerSet("new txt",
329 QStringList() << "txt",
330 QStringList() << "-o" << logName(logger: "txt")+",txt"
331 )
332 << LoggerSet("new stdout xml",
333 QStringList() << "stdout xml",
334 QStringList() << "-o" << "-,xml"
335 )
336 << LoggerSet("new xml",
337 QStringList() << "xml",
338 QStringList() << "-o" << logName(logger: "xml")+",xml"
339 )
340 << LoggerSet("new stdout junitxml",
341 QStringList() << "stdout junitxml",
342 QStringList() << "-o" << "-,junitxml"
343 )
344 << LoggerSet("new stdout xunitxml compatibility",
345 QStringList() << "stdout junitxml",
346 QStringList() << "-o" << "-,xunitxml"
347 )
348 << LoggerSet("new junitxml",
349 QStringList() << "junitxml",
350 QStringList() << "-o" << logName(logger: "junitxml")+",junitxml"
351 )
352 << LoggerSet("new stdout lightxml",
353 QStringList() << "stdout lightxml",
354 QStringList() << "-o" << "-,lightxml"
355 )
356 << LoggerSet("new lightxml",
357 QStringList() << "lightxml",
358 QStringList() << "-o" << logName(logger: "lightxml")+",lightxml"
359 )
360 << LoggerSet("new stdout csv", // benchmarks only
361 QStringList() << "stdout csv",
362 QStringList() << "-o" << "-,csv")
363 << LoggerSet("new csv", // benchmarks only
364 QStringList() << "csv",
365 QStringList() << "-o" << logName(logger: "csv")+",csv")
366 << LoggerSet("new stdout teamcity",
367 QStringList() << "stdout teamcity",
368 QStringList() << "-o" << "-,teamcity"
369 )
370 << LoggerSet("new teamcity",
371 QStringList() << "teamcity",
372 QStringList() << "-o" << logName(logger: "teamcity")+",teamcity"
373 )
374 << LoggerSet("new stdout tap",
375 QStringList() << "stdout tap",
376 QStringList() << "-o" << "-,tap"
377 )
378 << LoggerSet("new tap",
379 QStringList() << "tap",
380 QStringList() << "-o" << logName(logger: "tap")+",tap"
381 )
382 // Test with two loggers (don't test all 32 combinations, just a sample)
383 << LoggerSet("stdout txt + txt",
384 QStringList() << "stdout txt" << "txt",
385 QStringList() << "-o" << "-,txt"
386 << "-o" << logName(logger: "txt")+",txt"
387 )
388 << LoggerSet("xml + stdout txt",
389 QStringList() << "xml" << "stdout txt",
390 QStringList() << "-o" << logName(logger: "xml")+",xml"
391 << "-o" << "-,txt"
392 )
393 << LoggerSet("txt + junitxml",
394 QStringList() << "txt" << "junitxml",
395 QStringList() << "-o" << logName(logger: "txt")+",txt"
396 << "-o" << logName(logger: "junitxml")+",junitxml"
397 )
398 << LoggerSet("lightxml + stdout junitxml",
399 QStringList() << "lightxml" << "stdout junitxml",
400 QStringList() << "-o" << logName(logger: "lightxml")+",lightxml"
401 << "-o" << "-,junitxml"
402 )
403 // All loggers at the same time (except csv)
404 << LoggerSet("all loggers",
405 QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "junitxml" << "tap",
406 QStringList() << "-o" << logName(logger: "txt")+",txt"
407 << "-o" << logName(logger: "xml")+",xml"
408 << "-o" << logName(logger: "lightxml")+",lightxml"
409 << "-o" << "-,txt"
410 << "-o" << logName(logger: "junitxml")+",junitxml"
411 << "-o" << logName(logger: "teamcity")+",teamcity"
412 << "-o" << logName(logger: "tap")+",tap"
413 )
414 ;
415}
416
417tst_Selftests::tst_Selftests()
418 : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
419{}
420
421void tst_Selftests::initTestCase()
422{
423 QVERIFY2(tempDir.isValid(), qPrintable(tempDir.errorString()));
424 //Detect the location of the sub programs
425 QString subProgram = QLatin1String("pass/pass");
426#if defined(Q_OS_WIN)
427 subProgram = QLatin1String("pass/pass.exe");
428#endif
429 QString testdataDir = QFINDTESTDATA(subProgram);
430 if (testdataDir.lastIndexOf(s: subProgram) > 0)
431 testdataDir = testdataDir.left(n: testdataDir.lastIndexOf(s: subProgram));
432 else
433 testdataDir = QCoreApplication::applicationDirPath();
434 // chdir to our testdata path and execute helper apps relative to that.
435 QVERIFY2(QDir::setCurrent(testdataDir), qPrintable("Could not chdir to " + testdataDir));
436}
437
438void tst_Selftests::runSubTest_data()
439{
440 QTest::addColumn<QString>(name: "subdir");
441 QTest::addColumn<QStringList>(name: "loggers");
442 QTest::addColumn<QStringList>(name: "arguments");
443 QTest::addColumn<bool>(name: "crashes");
444
445 QStringList tests = QStringList()
446#if !defined(Q_OS_WIN)
447 // On windows, assert does nothing in release mode and blocks execution
448 // with a popup window in debug mode.
449 << "assert"
450#endif
451 << "badxml"
452#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
453 // Only run on platforms where callgrind is available.
454 << "benchlibcallgrind"
455#endif
456 << "benchlibcounting"
457 << "benchlibeventcounter"
458 << "benchliboptions"
459 << "blacklisted"
460 << "cmptest"
461 << "commandlinedata"
462 << "counting"
463 << "crashes"
464 << "datatable"
465 << "datetime"
466 << "differentexec"
467#if !defined(QT_NO_EXCEPTIONS) && !defined(Q_CC_INTEL) && !defined(Q_OS_WIN)
468 // Disable this test on Windows and for intel compiler, as the run-times
469 // will popup dialogs with warnings that uncaught exceptions were thrown
470 << "exceptionthrow"
471#endif
472 << "expectfail"
473 << "failcleanup"
474#ifndef Q_OS_WIN // these assert, by design; so same problem as "assert"
475 << "faildatatype"
476 << "failfetchtype"
477#endif
478 << "failinit"
479 << "failinitdata"
480#ifndef Q_OS_WIN // asserts, by design; so same problem as "assert"
481 << "fetchbogus"
482#endif
483 << "findtestdata"
484 << "float"
485 << "globaldata"
486 << "keyboard"
487 << "longstring"
488 << "maxwarnings"
489 << "multiexec"
490 << "pass"
491 << "pairdiagnostics"
492 << "printdatatags"
493 << "printdatatagswithglobaltags"
494 << "qexecstringlist"
495 << "signaldumper"
496 << "silent"
497 << "singleskip"
498 << "skip"
499 << "skipcleanup"
500 << "skipinit"
501 << "skipinitdata"
502 << "sleep"
503 << "strcmp"
504 << "subtest"
505 << "testlib"
506 << "tuplediagnostics"
507 << "verbose1"
508 << "verbose2"
509#ifndef QT_NO_EXCEPTIONS
510 // this test will test nothing if the exceptions are disabled
511 << "verifyexceptionthrown"
512#endif //!QT_NO_EXCEPTIONS
513 << "warnings"
514 << "watchdog"
515 << "xunit"
516 ;
517
518 // These tests are affected by timing and whether the CPU tick counter
519 // is monotonically increasing. They won't work on some machines so
520 // leave them off by default. Feel free to enable them for your own
521 // testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment
522 // variable to something non-empty.
523 if (!qgetenv(varName: "QTEST_ENABLE_EXTRA_SELFTESTS").isEmpty())
524 tests << "benchlibtickcounter"
525 << "benchlibwalltime"
526 ;
527
528 foreach (LoggerSet const& loggerSet, allLoggerSets()) {
529 QStringList loggers = loggerSet.loggers;
530
531 foreach (QString const& subtest, tests) {
532 // These tests don't work right unless logging plain text to
533 // standard output, either because they execute multiple test
534 // objects or because they internally supply arguments to
535 // themselves.
536 if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") {
537 if (subtest == "differentexec") {
538 continue;
539 }
540 if (subtest == "multiexec") {
541 continue;
542 }
543 if (subtest == "qexecstringlist") {
544 continue;
545 }
546 if (subtest == "benchliboptions") {
547 continue;
548 }
549 if (subtest == "blacklisted") {
550 continue;
551 }
552 if (subtest == "printdatatags") {
553 continue;
554 }
555 if (subtest == "printdatatagswithglobaltags") {
556 continue;
557 }
558 if (subtest == "silent") {
559 continue;
560 }
561 // `crashes' will not output valid XML on platforms without a crash handler
562 if (subtest == "crashes") {
563 continue;
564 }
565 // this test prints out some floats in the testlog and the formatting is
566 // platform-specific and hard to predict.
567 if (subtest == "float") {
568 continue;
569 }
570 // these tests are quite slow, and running them for all the loggers significantly
571 // increases the overall test time. They do not really relate to logging, so it
572 // should be safe to run them just for the stdout loggers.
573 if (subtest == "benchlibcallgrind" || subtest == "sleep") {
574 continue;
575 }
576 }
577 if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains(s: "txt")))
578 continue; // XML only, do not mix txt and XML for encoding test.
579
580 if (loggerSet.name.contains(s: "csv") && !subtest.startsWith(s: "benchlib"))
581 continue;
582
583 if (loggerSet.name.contains(s: "teamcity") && subtest.startsWith(s: "benchlib"))
584 continue; // Skip benchmark for TeamCity logger
585
586 // Keep in sync with generateTestData()'s crashers in generate_expected_output.py:
587 const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
588 || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
589 || subtest == QLatin1String("faildatatype") || subtest == QLatin1String("failfetchtype")
590 || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
591 || subtest == QLatin1String("blacklisted") || subtest == QLatin1String("watchdog");
592 QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
593 << subtest
594 << loggers
595 << loggerSet.arguments
596 << crashes
597 ;
598 }
599 }
600}
601
602#if QT_CONFIG(process)
603
604static QProcessEnvironment processEnvironment()
605{
606 static QProcessEnvironment result;
607 if (result.isEmpty()) {
608 const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
609 const bool preserveLibPath = qEnvironmentVariableIsSet(varName: "QT_PRESERVE_TESTLIB_PATH");
610 foreach (const QString &key, systemEnvironment.keys()) {
611 const bool useVariable = key == QLatin1String("PATH") || key == QLatin1String("QT_QPA_PLATFORM")
612#if defined(Q_OS_QNX)
613 || key == QLatin1String("GRAPHICS_ROOT") || key == QLatin1String("TZ")
614#elif defined(Q_OS_UNIX)
615 || key == QLatin1String("HOME") || key == QLatin1String("USER") // Required for X11 on openSUSE
616 || key == QLatin1String("QEMU_SET_ENV") || key == QLatin1String("QEMU_LD_PREFIX") // Required for QEMU
617# if !defined(Q_OS_MAC)
618 || key == QLatin1String("DISPLAY") || key == QLatin1String("XAUTHLOCALHOSTNAME")
619 || key.startsWith(s: QLatin1String("XDG_")) || key == QLatin1String("XAUTHORITY")
620# endif // !Q_OS_MAC
621#endif // Q_OS_UNIX
622#ifdef __COVERAGESCANNER__
623 || key == QLatin1String("QT_TESTCOCOON_ACTIVE")
624#endif
625 || ( preserveLibPath && (key == QLatin1String("QT_PLUGIN_PATH")
626 || key == QLatin1String("LD_LIBRARY_PATH")))
627 ;
628 if (useVariable)
629 result.insert(name: key, value: systemEnvironment.value(name: key));
630 }
631 // Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/:
632 result.insert(QStringLiteral("QT_LOGGING_RULES"),
633 // Must match generate_expected_output.py's main()'s value:
634 QStringLiteral("*.debug=true;qt.*=false"));
635 }
636 return result;
637}
638
639static inline QByteArray msgProcessError(const QString &binary, const QStringList &args,
640 const QProcessEnvironment &e, const QString &what)
641{
642 QString result;
643 QTextStream(&result) <<"Error running " << binary << ' ' << args.join(sep: ' ')
644 << " with environment " << e.toStringList().join(sep: ' ') << ": " << what;
645 return result.toLocal8Bit();
646}
647
648void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
649{
650#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
651 if (subdir == "benchlibcallgrind") {
652 QProcess checkProcess;
653 QStringList args;
654 args << QLatin1String("--version");
655 checkProcess.start(QLatin1String("valgrind"), args);
656 if (!checkProcess.waitForFinished(-1))
657 QSKIP(QString("Valgrind broken or not available. Not running %1 test!").arg(subdir).toLocal8Bit());
658 }
659#endif
660
661 QProcess proc;
662 QProcessEnvironment environment = processEnvironment();
663 // Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py:
664 if (crashes) {
665 environment.insert(name: "QTEST_DISABLE_CORE_DUMP", value: "1");
666 environment.insert(name: "QTEST_DISABLE_STACK_DUMP", value: "1");
667 if (subdir == QLatin1String("watchdog"))
668 environment.insert(name: "QTEST_FUNCTION_TIMEOUT", value: "100");
669 }
670 proc.setProcessEnvironment(environment);
671 const QString path = subdir + QLatin1Char('/') + subdir;
672 proc.start(program: path, arguments);
673 QVERIFY2(proc.waitForStarted(), msgProcessError(path, arguments, environment, QStringLiteral("Cannot start: ") + proc.errorString()));
674 QVERIFY2(proc.waitForFinished(), msgProcessError(path, arguments, environment, QStringLiteral("Timed out: ") + proc.errorString()));
675 if (!crashes) {
676 QVERIFY2(proc.exitStatus() == QProcess::NormalExit,
677 msgProcessError(path, arguments, environment,
678 QStringLiteral("Crashed: ") + proc.errorString()
679 + QStringLiteral(": ") + QString::fromLocal8Bit(proc.readAllStandardError())));
680 }
681
682 QList<QByteArray> actualOutputs;
683 for (int i = 0; i < loggers.count(); ++i) {
684 QString logFile = logName(logger: loggers[i]);
685 QByteArray out;
686 if (logFile.isEmpty()) {
687 out = proc.readAllStandardOutput();
688 } else {
689 QFile file(logFile);
690 if (file.open(flags: QIODevice::ReadOnly))
691 out = file.readAll();
692 }
693 actualOutputs << out;
694 }
695
696 const QByteArray err(proc.readAllStandardError());
697
698 // Some tests may output unpredictable strings to stderr, which we'll ignore.
699 //
700 // For instance, uncaught exceptions on Windows might say (depending on Windows
701 // version and JIT debugger settings):
702 // "This application has requested the Runtime to terminate it in an unusual way.
703 // Please contact the application's support team for more information."
704 //
705 // Also, tests which use valgrind may generate warnings if the toolchain is
706 // newer than the valgrind version, such that valgrind can't understand the
707 // debug information on the binary.
708 if (subdir != QLatin1String("exceptionthrow")
709 && subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication
710 && subdir != QLatin1String("fetchbogus")
711 && subdir != QLatin1String("watchdog")
712 && subdir != QLatin1String("xunit")
713#ifdef Q_CC_MINGW
714 && subdir != QLatin1String("blacklisted") // calls qFatal()
715 && subdir != QLatin1String("silent") // calls qFatal()
716#endif
717#ifdef Q_OS_LINUX
718 // QEMU outputs to stderr about uncaught signals
719 && !(EmulationDetector::isRunningArmOnX86() &&
720 (subdir == QLatin1String("assert")
721 || subdir == QLatin1String("blacklisted")
722 || subdir == QLatin1String("crashes")
723 || subdir == QLatin1String("faildatatype")
724 || subdir == QLatin1String("failfetchtype")
725 || subdir == QLatin1String("silent")
726 )
727 )
728#endif
729 && subdir != QLatin1String("benchlibcallgrind"))
730 QVERIFY2(err.isEmpty(), err.constData());
731
732 for (int n = 0; n < loggers.count(); ++n) {
733 QString logger = loggers[n];
734 if (n == 0 && subdir == QLatin1String("crashes")) {
735 QByteArray &actual = actualOutputs[0];
736#ifndef Q_OS_WIN
737 // Remove digits of times to match the expected file.
738 const QByteArray timePattern("Function time:");
739 int timePos = actual.indexOf(a: timePattern);
740 if (timePos >= 0) {
741 timePos += timePattern.size();
742 const int nextLinePos = actual.indexOf(c: '\n', from: timePos);
743 for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) {
744 if (actual.at(i: c) >= '0' && actual.at(i: c) <= '9')
745 actual.remove(index: c, len: 1);
746 }
747 }
748#else // !Q_OS_WIN
749 // Remove stack trace which is output to stdout.
750 const int exceptionLogStart = actual.indexOf("A crash occurred in ");
751 if (exceptionLogStart >= 0)
752 actual.truncate(exceptionLogStart);
753#endif // Q_OS_WIN
754 }
755
756 QList<QByteArray> res = splitLines(ba: actualOutputs[n]);
757 QString errorMessage;
758 QString expectedFileName = expectedFileNameFromTest(subdir, logger);
759 QByteArrayList exp = expectedResult(fileName: expectedFileName);
760 if (!exp.isEmpty()) {
761 if (!compareOutput(logger, subdir, rawOutput: actualOutputs[n], actual: res, expected: exp, errorMessage: &errorMessage)) {
762 errorMessage.prepend(s: QLatin1Char('"') + logger + QLatin1String("\", ")
763 + expectedFileName + QLatin1Char(' '));
764 errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
765 const QByteArray diff = runDiff(expected: exp, actual: res);
766 if (!diff.isEmpty())
767 errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
768 QFAIL(qPrintable(errorMessage));
769 }
770 } else {
771 // For the "crashes" and other tests, there are multiple versions of the
772 // expected output. Loop until a matching one is found.
773 bool ok = false;
774 for (int i = 1; !ok; ++i) {
775 expectedFileName = expectedFileNameFromTest(subdir: subdir + QLatin1Char('_') + QString::number(i), logger);
776 const QByteArrayList exp = expectedResult(fileName: expectedFileName);
777 if (exp.isEmpty())
778 break;
779 QString errorMessage2;
780 ok = compareOutput(logger, subdir, rawOutput: actualOutputs[n], actual: res, expected: exp, errorMessage: &errorMessage2);
781 if (!ok)
782 errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
783 }
784 if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
785 errorMessage.prepend(s: QLatin1String("Cannot find a matching file for ") + subdir);
786 errorMessage += QLatin1String("\nActual:\n");
787 QDebug(&errorMessage) << actualOutputs[n];
788 QFAIL(qPrintable(errorMessage));
789 }
790 }
791 }
792}
793
794static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
795static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
796
797bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
798 const QByteArray &rawOutput, const QByteArrayList &actual,
799 const QByteArrayList &expected,
800 QString *errorMessage) const
801{
802
803 if (actual.size() != expected.size()) {
804 *errorMessage = QString::fromLatin1(str: "Mismatch in line count. Expected %1 but got %2.")
805 .arg(a: expected.size()).arg(a: actual.size());
806 return false;
807 }
808
809 // For xml output formats, verify that the log is valid XML.
810 if (logger.endsWith(s: QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
811 return false;
812
813 // Verify that the actual output is an acceptable match for the
814 // expected output.
815
816 const QString qtVersion = QLatin1String(QT_VERSION_STR);
817 bool benchmark = false;
818 for (int i = 0, size = actual.size(); i < size; ++i) {
819 const QByteArray &actualLineBA = actual.at(i);
820 // the __FILE__ __LINE__ output is compiler dependent, skip it
821 if (actualLineBA.startsWith(c: " Loc: [") && actualLineBA.endsWith(c: ")]"))
822 continue;
823 if (actualLineBA.endsWith(c: " : failure location"))
824 continue;
825 if (actualLineBA.endsWith(c: " : message location"))
826 continue;
827
828 if (actualLineBA.startsWith(c: "Config: Using QtTest library") // Text build string
829 || actualLineBA.startsWith(c: " <QtBuild") // XML, Light XML build string
830 || (actualLineBA.startsWith(c: " <property value=") && actualLineBA.endsWith(c: "name=\"QtBuild\"/>"))) { // XUNIT-XML build string
831 continue;
832 }
833
834 QString actualLine = QString::fromLatin1(str: actualLineBA);
835 QString expectedLine = QString::fromLatin1(str: expected.at(i));
836 expectedLine.replace(before: qtVersionPlaceHolder(), after: qtVersion);
837
838 // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
839 if (logger.endsWith(s: QLatin1String("teamcity"))) {
840 static QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]");
841 actualLine.replace(re: teamcityLocRegExp, after: teamCityLocation());
842 expectedLine.replace(re: teamcityLocRegExp, after: teamCityLocation());
843 }
844
845 if (logger.endsWith(s: QLatin1String("tap"))) {
846 if (expectedLine.contains(s: QLatin1String("at:"))
847 || expectedLine.contains(s: QLatin1String("file:"))
848 || expectedLine.contains(s: QLatin1String("line:")))
849 actualLine = expectedLine;
850 }
851
852 if (!compareLine(logger, subdir, benchmark, actualLine,
853 expLine: expectedLine, errorMessage)) {
854 errorMessage->prepend(s: QLatin1String("Line ") + QString::number(i + 1)
855 + QLatin1String(": "));
856 return false;
857 }
858
859 benchmark = actualLineBA.startsWith(c: "RESULT : ");
860 }
861 return true;
862}
863
864bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
865 bool benchmark,
866 const QString &actualLine, const QString &expectedLine,
867 QString *errorMessage) const
868{
869 if (actualLine == expectedLine)
870 return true;
871
872 if ((subdir == QLatin1String("assert")
873 || subdir == QLatin1String("faildatatype") || subdir == QLatin1String("failfetchtype"))
874 && actualLine.contains(s: QLatin1String("ASSERT: "))
875 && expectedLine.contains(s: QLatin1String("ASSERT: "))) {
876 // Q_ASSERT uses __FILE__, the exact contents of which are
877 // undefined. If have we something that looks like a Q_ASSERT and we
878 // were expecting to see a Q_ASSERT, we'll skip the line.
879 return true;
880 }
881
882 if (expectedLine.startsWith(s: QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))) {
883 // On some platforms we compile without RTTI, and as a result we never throw an exception
884 if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
885 *errorMessage = QString::fromLatin1(str: "'%1' != 'tst_Exception::throwException()'").arg(a: actualLine);
886 return false;
887 }
888 return true;
889 }
890
891 if (benchmark || actualLine.startsWith(s: QLatin1String("<BenchmarkResult"))
892 || (logger == QLatin1String("csv") && actualLine.startsWith(c: QLatin1Char('"')))) {
893 // Don't do a literal comparison for benchmark results, since
894 // results have some natural variance.
895 QString error;
896 BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
897 if (!error.isEmpty()) {
898 *errorMessage = QString::fromLatin1(str: "Actual line didn't parse as benchmark result: %1\nLine: %2").arg(args&: error, args: actualLine);
899 return false;
900 }
901 BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
902 if (!error.isEmpty()) {
903 *errorMessage = QString::fromLatin1(str: "Expected line didn't parse as benchmark result: %1\nLine: %2").arg(args&: error, args: expectedLine);
904 return false;
905 }
906 return compareBenchmarkResult(r1: actualResult, r2: expectedResult, errorMessage);
907 }
908
909 if (actualLine.startsWith(s: QLatin1String(" <Duration msecs="))
910 || actualLine.startsWith(s: QLatin1String("<Duration msecs="))) {
911 static QRegularExpression durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>");
912 QRegularExpressionMatch match = durationRegExp.match(subject: actualLine);
913 if (match.hasMatch())
914 return true;
915 *errorMessage = QString::fromLatin1(str: "Invalid Duration tag: '%1'").arg(a: actualLine);
916 return false;
917 }
918
919 if (actualLine.startsWith(s: QLatin1String("Totals:")) && expectedLine.startsWith(s: QLatin1String("Totals:")))
920 return true;
921
922 const QLatin1String pointerPlaceholder("_POINTER_");
923 if (expectedLine.contains(s: pointerPlaceholder)
924 && (expectedLine.contains(s: QLatin1String("Signal: "))
925 || expectedLine.contains(s: QLatin1String("Slot: ")))) {
926 QString actual = actualLine;
927 // We don't care about the pointer of the object to whom the signal belongs, so we
928 // replace it with _POINTER_, e.g.:
929 // Signal: SignalSlotClass(7ffd72245410) signalWithoutParameters ()
930 // Signal: QThread(7ffd72245410) started ()
931 // After this instance pointer we may have further pointers and
932 // references (with an @ prefix) as parameters of the signal or
933 // slot being invoked.
934 // Signal: SignalSlotClass(_POINTER_) qStringRefSignal ((QString&)@55f5fbb8dd40)
935 actual.replace(re: QRegularExpression("\\b[a-f0-9]{8,}\\b"), after: pointerPlaceholder);
936 // Also change QEventDispatcher{Glib,Win32,etc.} to QEventDispatcherPlatform
937 actual.replace(re: QRegularExpression("\\b(QEventDispatcher)\\w+\\b"), after: QLatin1String("\\1Platform"));
938 if (actual != expectedLine) {
939 *errorMessage = msgMismatch(actual, expected: expectedLine);
940 return false;
941 }
942 return true;
943 }
944
945 if (EmulationDetector::isRunningArmOnX86() && subdir == QLatin1String("float")) {
946 // QEMU cheats at qfloat16, so outputs it as if it were a float.
947 if (actualLine.endsWith(s: QLatin1String("Actual (operandLeft) : 0.001"))
948 && expectedLine.endsWith(s: QLatin1String("Actual (operandLeft) : 0.000999"))) {
949 return true;
950 }
951 }
952
953 *errorMessage = msgMismatch(actual: actualLine, expected: expectedLine);
954 return false;
955}
956
957bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
958 QString *errorMessage) const
959{
960 // lightxml intentionally skips the root element, which technically makes it
961 // not valid XML.
962 // We'll add that ourselves for the purpose of validation.
963 if (logger.endsWith(s: QLatin1String("lightxml"))) {
964 xml.prepend(s: "<root>");
965 xml.append(s: "</root>");
966 }
967
968 QXmlStreamReader reader(xml);
969 while (!reader.atEnd())
970 reader.readNext();
971
972 if (reader.hasError()) {
973 const int lineNumber = int(reader.lineNumber());
974 const QByteArray line = xml.split(sep: '\n').value(i: lineNumber - 1);
975 *errorMessage = QString::fromLatin1(str: "line %1, col %2 '%3': %4")
976 .arg(a: lineNumber).arg(a: reader.columnNumber())
977 .arg(args: QString::fromLatin1(str: line), args: reader.errorString());
978 return false;
979 }
980 return true;
981}
982
983#endif // QT_CONFIG(process)
984
985void tst_Selftests::runSubTest()
986{
987#if !QT_CONFIG(process)
988 QSKIP("This test requires QProcess support");
989#else
990 QFETCH(QString, subdir);
991 QFETCH(QStringList, loggers);
992 QFETCH(QStringList, arguments);
993 QFETCH(bool, crashes);
994
995 doRunSubTest(subdir, loggers, arguments, crashes);
996#endif // QT_CONFIG(process)
997}
998
999// attribute must contain ="
1000QString extractXmlAttribute(const QString &line, const char *attribute)
1001{
1002 int index = line.indexOf(s: attribute);
1003 if (index == -1)
1004 return QString();
1005 const int attributeLength = int(strlen(s: attribute));
1006 const int end = line.indexOf(c: '"', from: index + attributeLength);
1007 if (end == -1)
1008 return QString();
1009
1010 const QString result = line.mid(position: index + attributeLength, n: end - index - attributeLength);
1011 if (result.isEmpty())
1012 return ""; // ensure empty but not null
1013 return result;
1014}
1015
1016// Parse line into the BenchmarkResult it represents.
1017BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
1018{
1019 if (error) *error = QString();
1020 BenchmarkResult out;
1021
1022 QString remaining = line.trimmed();
1023
1024 if (remaining.isEmpty()) {
1025 if (error) *error = "Line is empty";
1026 return out;
1027 }
1028
1029 if (line.startsWith(s: "<BenchmarkResult ")) {
1030 // XML result
1031 // format:
1032 // <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" />
1033 if (!line.endsWith(s: "/>")) {
1034 if (error) *error = "unterminated XML";
1035 return out;
1036 }
1037
1038 QString unit = extractXmlAttribute(line, attribute: " metric=\"");
1039 QString sTotal = extractXmlAttribute(line, attribute: " value=\"");
1040 QString sIterations = extractXmlAttribute(line, attribute: " iterations=\"");
1041 if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) {
1042 if (error) *error = "XML snippet did not contain all required values";
1043 return out;
1044 }
1045
1046 bool ok;
1047 double total = sTotal.toDouble(ok: &ok);
1048 if (!ok) {
1049 if (error) *error = sTotal + " is not a valid number";
1050 return out;
1051 }
1052 double iterations = sIterations.toDouble(ok: &ok);
1053 if (!ok) {
1054 if (error) *error = sIterations + " is not a valid number";
1055 return out;
1056 }
1057
1058 out.unit = unit;
1059 out.total = total;
1060 out.iterations = iterations;
1061 return out;
1062 }
1063
1064 if (line.startsWith(c: '"')) {
1065 // CSV result
1066 // format:
1067 // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
1068 QStringList split = line.split(sep: ',');
1069 if (split.count() != 6) {
1070 if (error) *error = QString("Wrong number of columns (%1)").arg(a: split.count());
1071 return out;
1072 }
1073
1074 bool ok;
1075 double total = split.at(i: 4).toDouble(ok: &ok);
1076 if (!ok) {
1077 if (error) *error = split.at(i: 4) + " is not a valid number";
1078 return out;
1079 }
1080 double iterations = split.at(i: 5).toDouble(ok: &ok);
1081 if (!ok) {
1082 if (error) *error = split.at(i: 5) + " is not a valid number";
1083 return out;
1084 }
1085
1086 out.unit = split.at(i: 2);
1087 out.total = total;
1088 out.iterations = iterations;
1089 return out;
1090 }
1091
1092 // Text result
1093 // This code avoids using a QRegExp because QRegExp might be broken.
1094 // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)
1095
1096 QString sFirstNumber;
1097 while (!remaining.isEmpty() && !remaining.at(i: 0).isSpace()) {
1098 sFirstNumber += remaining.at(i: 0);
1099 remaining.remove(i: 0,len: 1);
1100 }
1101 remaining = remaining.trimmed();
1102
1103 // 4,000 -> 4000
1104 sFirstNumber.remove(c: ',');
1105
1106 // Should now be parseable as floating point
1107 bool ok;
1108 double firstNumber = sFirstNumber.toDouble(ok: &ok);
1109 if (!ok) {
1110 if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
1111 return out;
1112 }
1113
1114 // Remaining: msec per iteration (total: 4000, iterations: 1)
1115 static const char periterbit[] = " per iteration (total: ";
1116 QString unit;
1117 while (!remaining.startsWith(s: periterbit) && !remaining.isEmpty()) {
1118 unit += remaining.at(i: 0);
1119 remaining.remove(i: 0,len: 1);
1120 }
1121 if (remaining.isEmpty()) {
1122 if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
1123 return out;
1124 }
1125
1126 remaining = remaining.mid(position: sizeof(periterbit)-1);
1127
1128 // Remaining: 4,000, iterations: 1)
1129 static const char itersbit[] = ", iterations: ";
1130 QString sTotal;
1131 while (!remaining.startsWith(s: itersbit) && !remaining.isEmpty()) {
1132 sTotal += remaining.at(i: 0);
1133 remaining.remove(i: 0,len: 1);
1134 }
1135 if (remaining.isEmpty()) {
1136 if (error) *error = "Could not find pattern: '<number>, iterations: '";
1137 return out;
1138 }
1139
1140 remaining = remaining.mid(position: sizeof(itersbit)-1);
1141
1142 // 4,000 -> 4000
1143 sTotal.remove(c: ',');
1144
1145 double total = sTotal.toDouble(ok: &ok);
1146 if (!ok) {
1147 if (error) *error = sTotal + " (total) is not a valid number";
1148 return out;
1149 }
1150
1151 // Remaining: 1)
1152 QString sIters;
1153 while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
1154 sIters += remaining.at(i: 0);
1155 remaining.remove(i: 0,len: 1);
1156 }
1157 if (remaining.isEmpty()) {
1158 if (error) *error = "Could not find pattern: '<num>)'";
1159 return out;
1160 }
1161 qint64 iters = sIters.toLongLong(ok: &ok);
1162 if (!ok) {
1163 if (error) *error = sIters + " (iterations) is not a valid integer";
1164 return out;
1165 }
1166
1167 double calcFirstNumber = double(total)/double(iters);
1168 if (!qFuzzyCompare(p1: firstNumber, p2: calcFirstNumber)) {
1169 if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(a: calcFirstNumber).arg(a: firstNumber);
1170 return out;
1171 }
1172
1173 out.total = total;
1174 out.unit = unit;
1175 out.iterations = iters;
1176 return out;
1177}
1178
1179void tst_Selftests::cleanup()
1180{
1181 QFETCH(QStringList, loggers);
1182
1183 // Remove the test output files
1184 for (int i = 0; i < loggers.count(); ++i) {
1185 QString logFileName = logName(logger: loggers[i]);
1186 if (!logFileName.isEmpty()) {
1187 QFile logFile(logFileName);
1188 if (logFile.exists())
1189 QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString())));
1190 }
1191 }
1192}
1193
1194QTEST_MAIN(tst_Selftests)
1195
1196#include "tst_selftests.moc"
1197

source code of qtbase/tests/auto/testlib/selftests/tst_selftests.cpp