1 | /**************************************************************************** |
2 | ** |
3 | ** Copyright (C) 2020 The Qt Company Ltd. |
4 | ** Copyright (C) 2016 Intel Corporation. |
5 | ** Contact: https://www.qt.io/licensing/ |
6 | ** |
7 | ** This file is part of the test suite of the Qt Toolkit. |
8 | ** |
9 | ** $QT_BEGIN_LICENSE:GPL-EXCEPT$ |
10 | ** Commercial License Usage |
11 | ** Licensees holding valid commercial Qt licenses may use this file in |
12 | ** accordance with the commercial license agreement provided with the |
13 | ** Software or, alternatively, in accordance with the terms contained in |
14 | ** a written agreement between you and The Qt Company. For licensing terms |
15 | ** and conditions see https://www.qt.io/terms-conditions. For further |
16 | ** information use the contact form at https://www.qt.io/contact-us. |
17 | ** |
18 | ** GNU General Public License Usage |
19 | ** Alternatively, this file may be used under the terms of the GNU |
20 | ** General Public License version 3 as published by the Free Software |
21 | ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT |
22 | ** included in the packaging of this file. Please review the following |
23 | ** information to ensure the GNU General Public License requirements will |
24 | ** be met: https://www.gnu.org/licenses/gpl-3.0.html. |
25 | ** |
26 | ** $QT_END_LICENSE$ |
27 | ** |
28 | ****************************************************************************/ |
29 | |
30 | #include <QtCore/QCoreApplication> |
31 | |
32 | #if QT_CONFIG(temporaryfile) && QT_CONFIG(process) |
33 | # define USE_DIFF |
34 | #endif |
35 | #include <QtCore/QXmlStreamReader> |
36 | #include <QtCore/QFileInfo> |
37 | #include <QtCore/QDir> |
38 | #include <QtCore/QTemporaryDir> |
39 | #ifdef USE_DIFF |
40 | # include <QtCore/QTemporaryFile> |
41 | # include <QtCore/QStandardPaths> |
42 | #endif |
43 | #include <QtTest/QtTest> |
44 | |
45 | #include <private/cycle_p.h> |
46 | |
47 | #include "emulationdetector.h" |
48 | |
49 | struct LoggerSet; |
50 | |
51 | class tst_Selftests: public QObject |
52 | { |
53 | Q_OBJECT |
54 | public: |
55 | tst_Selftests(); |
56 | |
57 | private slots: |
58 | void initTestCase(); |
59 | void runSubTest_data(); |
60 | void runSubTest(); |
61 | void cleanup(); |
62 | |
63 | private: |
64 | void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes); |
65 | bool compareOutput(const QString &logger, const QString &subdir, |
66 | const QByteArray &rawOutput, const QByteArrayList &actual, |
67 | const QByteArrayList &expected, |
68 | QString *errorMessage) const; |
69 | bool compareLine(const QString &logger, const QString &subdir, bool benchmark, |
70 | const QString &actualLine, const QString &expLine, |
71 | QString *errorMessage) const; |
72 | bool checkXml(const QString &logger, QByteArray rawOutput, |
73 | QString *errorMessage) const; |
74 | |
75 | QString logName(const QString &logger) const; |
76 | QList<LoggerSet> allLoggerSets() const; |
77 | |
78 | QTemporaryDir tempDir; |
79 | }; |
80 | |
81 | struct BenchmarkResult |
82 | { |
83 | qint64 total; |
84 | qint64 iterations; |
85 | QString unit; |
86 | |
87 | inline QString toString() const |
88 | { return QString("total:%1, unit:%2, iterations:%3" ).arg(a: total).arg(a: unit).arg(a: iterations); } |
89 | |
90 | static BenchmarkResult parse(QString const&, QString*); |
91 | }; |
92 | |
93 | static QString msgMismatch(const QString &actual, const QString &expected) |
94 | { |
95 | return QLatin1String("Mismatch:\n'" ) + actual + QLatin1String("'\n !=\n'" ) |
96 | + expected + QLatin1Char('\''); |
97 | } |
98 | |
99 | static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2, |
100 | QString *errorMessage) |
101 | { |
102 | // First make sure the iterations and unit match. |
103 | if (r1.iterations != r2.iterations || r1.unit != r2.unit) { |
104 | // Nope - compare whole string for best failure message |
105 | *errorMessage = msgMismatch(actual: r1.toString(), expected: r2.toString()); |
106 | return false; |
107 | } |
108 | |
109 | // Now check the value. Some variance is allowed, and how much depends on |
110 | // the measured unit. |
111 | qreal variance = 0.; |
112 | if (r1.unit == QLatin1String("msecs" ) || r1.unit == QLatin1String("WalltimeMilliseconds" )) |
113 | variance = 0.1; |
114 | else if (r1.unit == QLatin1String("instruction reads" )) |
115 | variance = 0.001; |
116 | else if (r1.unit == QLatin1String("CPU ticks" ) || r1.unit == QLatin1String("CPUTicks" )) |
117 | variance = 0.001; |
118 | |
119 | if (variance == 0.) { |
120 | // No variance allowed - compare whole string |
121 | const QString r1S = r1.toString(); |
122 | const QString r2S = r2.toString(); |
123 | if (r1S != r2S) { |
124 | *errorMessage = msgMismatch(actual: r1S, expected: r2S); |
125 | return false; |
126 | } |
127 | return true; |
128 | } |
129 | |
130 | if (qAbs(t: qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) { |
131 | // Whoops, didn't match. Compare the whole string for the most useful failure message. |
132 | *errorMessage = msgMismatch(actual: r1.toString(), expected: r2.toString()); |
133 | return false; |
134 | } |
135 | return true; |
136 | } |
137 | |
138 | // Split the passed block of text into an array of lines, replacing any |
139 | // filenames and line numbers with generic markers to avoid failing the test |
140 | // due to compiler-specific behaviour. |
141 | static QList<QByteArray> splitLines(QByteArray ba) |
142 | { |
143 | ba.replace(before: '\r', c: "" ); |
144 | QList<QByteArray> out = ba.split(sep: '\n'); |
145 | |
146 | // Replace any ` file="..."' or ` line="..."' in XML with a generic location. |
147 | static const char *markers[][2] = { |
148 | { " file=\"" , " file=\"__FILE__\"" }, |
149 | { " line=\"" , " line=\"__LINE__\"" } |
150 | }; |
151 | static const int markerCount = sizeof markers / sizeof markers[0]; |
152 | |
153 | for (int i = 0; i < out.size(); ++i) { |
154 | QByteArray& line = out[i]; |
155 | for (int j = 0; j < markerCount; ++j) { |
156 | int index = line.indexOf(c: markers[j][0]); |
157 | if (index == -1) { |
158 | continue; |
159 | } |
160 | const int end = line.indexOf(c: '"', from: index + int(strlen(s: markers[j][0]))); |
161 | if (end == -1) { |
162 | continue; |
163 | } |
164 | line.replace(index, len: end-index + 1, s: markers[j][1]); |
165 | } |
166 | } |
167 | |
168 | return out; |
169 | } |
170 | |
171 | // Return the log format, e.g. for both "stdout txt" and "txt", return "txt'. |
172 | static inline QString logFormat(const QString &logger) |
173 | { |
174 | return (logger.startsWith(s: "stdout" ) ? logger.mid(position: 7) : logger); |
175 | } |
176 | |
177 | // Return the log file name, or an empty string if the log goes to stdout. |
178 | QString tst_Selftests::logName(const QString &logger) const |
179 | { |
180 | return (logger.startsWith(s: "stdout" ) ? "" : QString(tempDir.path() + "/test_output." + logger)); |
181 | } |
182 | |
183 | static QString expectedFileNameFromTest(const QString &subdir, const QString &logger) |
184 | { |
185 | return QStringLiteral("expected_" ) + subdir + QLatin1Char('.') + logFormat(logger); |
186 | } |
187 | |
188 | // Load the expected test output for the nominated test (subdir) and logger |
189 | // as an array of lines. If there is no expected output file, return an |
190 | // empty array. |
191 | static QList<QByteArray> expectedResult(const QString &fileName) |
192 | { |
193 | QFile file(QStringLiteral(":/" ) + fileName); |
194 | if (!file.open(flags: QIODevice::ReadOnly)) |
195 | return QList<QByteArray>(); |
196 | return splitLines(ba: file.readAll()); |
197 | } |
198 | |
199 | // Helpers for running the 'diff' tool in case comparison fails |
200 | #ifdef USE_DIFF |
201 | static inline void writeLines(QIODevice &d, const QByteArrayList &lines) |
202 | { |
203 | for (const QByteArray &l : lines) { |
204 | d.write(data: l); |
205 | d.write(data: "\n" ); |
206 | } |
207 | } |
208 | #endif // USE_DIFF |
209 | |
210 | static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual) |
211 | { |
212 | QByteArray result; |
213 | #ifdef USE_DIFF |
214 | # ifndef Q_OS_WIN |
215 | const QString diff = QStandardPaths::findExecutable(executableName: "diff" ); |
216 | # else |
217 | const QString diff = QStandardPaths::findExecutable("diff.exe" ); |
218 | # endif |
219 | if (diff.isEmpty()) |
220 | return result; |
221 | QTemporaryFile expectedFile; |
222 | if (!expectedFile.open()) |
223 | return result; |
224 | writeLines(d&: expectedFile, lines: expected); |
225 | expectedFile.close(); |
226 | QTemporaryFile actualFile; |
227 | if (!actualFile.open()) |
228 | return result; |
229 | writeLines(d&: actualFile, lines: actual); |
230 | actualFile.close(); |
231 | QProcess diffProcess; |
232 | diffProcess.start(program: diff, arguments: {QLatin1String("-u" ), expectedFile.fileName(), actualFile.fileName()}); |
233 | if (!diffProcess.waitForStarted()) |
234 | return result; |
235 | if (diffProcess.waitForFinished()) |
236 | result = diffProcess.readAllStandardOutput(); |
237 | else |
238 | diffProcess.kill(); |
239 | #endif // USE_DIFF |
240 | return result; |
241 | } |
242 | |
243 | // Each test is run with a set of one or more test output loggers. |
244 | // This struct holds information about one such test. |
245 | struct LoggerSet |
246 | { |
247 | LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments) |
248 | : name(_name), loggers(_loggers), arguments(_arguments) |
249 | { } |
250 | |
251 | QString name; |
252 | QStringList loggers; |
253 | QStringList arguments; |
254 | }; |
255 | |
256 | // This function returns a list of all sets of loggers to be used for |
257 | // running each subtest. |
258 | QList<LoggerSet> tst_Selftests::allLoggerSets() const |
259 | { |
260 | // Note that in order to test XML output to standard output, the subtests |
261 | // must not send output directly to stdout, bypassing Qt's output mechanisms |
262 | // (e.g. via printf), otherwise the output may not be well-formed XML. |
263 | return QList<LoggerSet>() |
264 | // Test with old-style options for a single logger |
265 | << LoggerSet("old stdout txt" , |
266 | QStringList() << "stdout txt" , |
267 | QStringList() |
268 | ) |
269 | << LoggerSet("old txt" , |
270 | QStringList() << "txt" , |
271 | QStringList() << "-o" << logName(logger: "txt" ) |
272 | ) |
273 | << LoggerSet("old stdout xml" , |
274 | QStringList() << "stdout xml" , |
275 | QStringList() << "-xml" |
276 | ) |
277 | << LoggerSet("old xml" , |
278 | QStringList() << "xml" , |
279 | QStringList() << "-xml" << "-o" << logName(logger: "xml" ) |
280 | ) |
281 | << LoggerSet("old stdout junitxml" , |
282 | QStringList() << "stdout junitxml" , |
283 | QStringList() << "-junitxml" |
284 | ) |
285 | << LoggerSet("old junitxml" , |
286 | QStringList() << "junitxml" , |
287 | QStringList() << "-junitxml" << "-o" << logName(logger: "junitxml" ) |
288 | ) |
289 | << LoggerSet("old xunitxml compatibility" , |
290 | QStringList() << "junitxml" , |
291 | QStringList() << "-xunitxml" << "-o" << logName(logger: "junitxml" ) |
292 | ) |
293 | << LoggerSet("old stdout lightxml" , |
294 | QStringList() << "stdout lightxml" , |
295 | QStringList() << "-lightxml" |
296 | ) |
297 | << LoggerSet("old lightxml" , |
298 | QStringList() << "lightxml" , |
299 | QStringList() << "-lightxml" << "-o" << logName(logger: "lightxml" ) |
300 | ) |
301 | << LoggerSet("old stdout csv" , // benchmarks only |
302 | QStringList() << "stdout csv" , |
303 | QStringList() << "-csv" ) |
304 | << LoggerSet("old csv" , // benchmarks only |
305 | QStringList() << "csv" , |
306 | QStringList() << "-csv" << "-o" << logName(logger: "csv" )) |
307 | << LoggerSet("old stdout teamcity" , |
308 | QStringList() << "stdout teamcity" , |
309 | QStringList() << "-teamcity" |
310 | ) |
311 | << LoggerSet("old teamcity" , |
312 | QStringList() << "teamcity" , |
313 | QStringList() << "-teamcity" << "-o" << logName(logger: "teamcity" ) |
314 | ) |
315 | << LoggerSet("old stdout tap" , |
316 | QStringList() << "stdout tap" , |
317 | QStringList() << "-tap" |
318 | ) |
319 | << LoggerSet("old tap" , |
320 | QStringList() << "tap" , |
321 | QStringList() << "-tap" << "-o" << logName(logger: "tap" ) |
322 | ) |
323 | // Test with new-style options for a single logger |
324 | << LoggerSet("new stdout txt" , |
325 | QStringList() << "stdout txt" , |
326 | QStringList() << "-o" << "-,txt" |
327 | ) |
328 | << LoggerSet("new txt" , |
329 | QStringList() << "txt" , |
330 | QStringList() << "-o" << logName(logger: "txt" )+",txt" |
331 | ) |
332 | << LoggerSet("new stdout xml" , |
333 | QStringList() << "stdout xml" , |
334 | QStringList() << "-o" << "-,xml" |
335 | ) |
336 | << LoggerSet("new xml" , |
337 | QStringList() << "xml" , |
338 | QStringList() << "-o" << logName(logger: "xml" )+",xml" |
339 | ) |
340 | << LoggerSet("new stdout junitxml" , |
341 | QStringList() << "stdout junitxml" , |
342 | QStringList() << "-o" << "-,junitxml" |
343 | ) |
344 | << LoggerSet("new stdout xunitxml compatibility" , |
345 | QStringList() << "stdout junitxml" , |
346 | QStringList() << "-o" << "-,xunitxml" |
347 | ) |
348 | << LoggerSet("new junitxml" , |
349 | QStringList() << "junitxml" , |
350 | QStringList() << "-o" << logName(logger: "junitxml" )+",junitxml" |
351 | ) |
352 | << LoggerSet("new stdout lightxml" , |
353 | QStringList() << "stdout lightxml" , |
354 | QStringList() << "-o" << "-,lightxml" |
355 | ) |
356 | << LoggerSet("new lightxml" , |
357 | QStringList() << "lightxml" , |
358 | QStringList() << "-o" << logName(logger: "lightxml" )+",lightxml" |
359 | ) |
360 | << LoggerSet("new stdout csv" , // benchmarks only |
361 | QStringList() << "stdout csv" , |
362 | QStringList() << "-o" << "-,csv" ) |
363 | << LoggerSet("new csv" , // benchmarks only |
364 | QStringList() << "csv" , |
365 | QStringList() << "-o" << logName(logger: "csv" )+",csv" ) |
366 | << LoggerSet("new stdout teamcity" , |
367 | QStringList() << "stdout teamcity" , |
368 | QStringList() << "-o" << "-,teamcity" |
369 | ) |
370 | << LoggerSet("new teamcity" , |
371 | QStringList() << "teamcity" , |
372 | QStringList() << "-o" << logName(logger: "teamcity" )+",teamcity" |
373 | ) |
374 | << LoggerSet("new stdout tap" , |
375 | QStringList() << "stdout tap" , |
376 | QStringList() << "-o" << "-,tap" |
377 | ) |
378 | << LoggerSet("new tap" , |
379 | QStringList() << "tap" , |
380 | QStringList() << "-o" << logName(logger: "tap" )+",tap" |
381 | ) |
382 | // Test with two loggers (don't test all 32 combinations, just a sample) |
383 | << LoggerSet("stdout txt + txt" , |
384 | QStringList() << "stdout txt" << "txt" , |
385 | QStringList() << "-o" << "-,txt" |
386 | << "-o" << logName(logger: "txt" )+",txt" |
387 | ) |
388 | << LoggerSet("xml + stdout txt" , |
389 | QStringList() << "xml" << "stdout txt" , |
390 | QStringList() << "-o" << logName(logger: "xml" )+",xml" |
391 | << "-o" << "-,txt" |
392 | ) |
393 | << LoggerSet("txt + junitxml" , |
394 | QStringList() << "txt" << "junitxml" , |
395 | QStringList() << "-o" << logName(logger: "txt" )+",txt" |
396 | << "-o" << logName(logger: "junitxml" )+",junitxml" |
397 | ) |
398 | << LoggerSet("lightxml + stdout junitxml" , |
399 | QStringList() << "lightxml" << "stdout junitxml" , |
400 | QStringList() << "-o" << logName(logger: "lightxml" )+",lightxml" |
401 | << "-o" << "-,junitxml" |
402 | ) |
403 | // All loggers at the same time (except csv) |
404 | << LoggerSet("all loggers" , |
405 | QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "junitxml" << "tap" , |
406 | QStringList() << "-o" << logName(logger: "txt" )+",txt" |
407 | << "-o" << logName(logger: "xml" )+",xml" |
408 | << "-o" << logName(logger: "lightxml" )+",lightxml" |
409 | << "-o" << "-,txt" |
410 | << "-o" << logName(logger: "junitxml" )+",junitxml" |
411 | << "-o" << logName(logger: "teamcity" )+",teamcity" |
412 | << "-o" << logName(logger: "tap" )+",tap" |
413 | ) |
414 | ; |
415 | } |
416 | |
417 | tst_Selftests::tst_Selftests() |
418 | : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX" ) |
419 | {} |
420 | |
421 | void tst_Selftests::initTestCase() |
422 | { |
423 | QVERIFY2(tempDir.isValid(), qPrintable(tempDir.errorString())); |
424 | //Detect the location of the sub programs |
425 | QString subProgram = QLatin1String("pass/pass" ); |
426 | #if defined(Q_OS_WIN) |
427 | subProgram = QLatin1String("pass/pass.exe" ); |
428 | #endif |
429 | QString testdataDir = QFINDTESTDATA(subProgram); |
430 | if (testdataDir.lastIndexOf(s: subProgram) > 0) |
431 | testdataDir = testdataDir.left(n: testdataDir.lastIndexOf(s: subProgram)); |
432 | else |
433 | testdataDir = QCoreApplication::applicationDirPath(); |
434 | // chdir to our testdata path and execute helper apps relative to that. |
435 | QVERIFY2(QDir::setCurrent(testdataDir), qPrintable("Could not chdir to " + testdataDir)); |
436 | } |
437 | |
438 | void tst_Selftests::runSubTest_data() |
439 | { |
440 | QTest::addColumn<QString>(name: "subdir" ); |
441 | QTest::addColumn<QStringList>(name: "loggers" ); |
442 | QTest::addColumn<QStringList>(name: "arguments" ); |
443 | QTest::addColumn<bool>(name: "crashes" ); |
444 | |
445 | QStringList tests = QStringList() |
446 | #if !defined(Q_OS_WIN) |
447 | // On windows, assert does nothing in release mode and blocks execution |
448 | // with a popup window in debug mode. |
449 | << "assert" |
450 | #endif |
451 | << "badxml" |
452 | #if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX) |
453 | // Only run on platforms where callgrind is available. |
454 | << "benchlibcallgrind" |
455 | #endif |
456 | << "benchlibcounting" |
457 | << "benchlibeventcounter" |
458 | << "benchliboptions" |
459 | << "blacklisted" |
460 | << "cmptest" |
461 | << "commandlinedata" |
462 | << "counting" |
463 | << "crashes" |
464 | << "datatable" |
465 | << "datetime" |
466 | << "differentexec" |
467 | #if !defined(QT_NO_EXCEPTIONS) && !defined(Q_CC_INTEL) && !defined(Q_OS_WIN) |
468 | // Disable this test on Windows and for intel compiler, as the run-times |
469 | // will popup dialogs with warnings that uncaught exceptions were thrown |
470 | << "exceptionthrow" |
471 | #endif |
472 | << "expectfail" |
473 | << "failcleanup" |
474 | #ifndef Q_OS_WIN // these assert, by design; so same problem as "assert" |
475 | << "faildatatype" |
476 | << "failfetchtype" |
477 | #endif |
478 | << "failinit" |
479 | << "failinitdata" |
480 | #ifndef Q_OS_WIN // asserts, by design; so same problem as "assert" |
481 | << "fetchbogus" |
482 | #endif |
483 | << "findtestdata" |
484 | << "float" |
485 | << "globaldata" |
486 | << "keyboard" |
487 | << "longstring" |
488 | << "maxwarnings" |
489 | << "multiexec" |
490 | << "pass" |
491 | << "pairdiagnostics" |
492 | << "printdatatags" |
493 | << "printdatatagswithglobaltags" |
494 | << "qexecstringlist" |
495 | << "signaldumper" |
496 | << "silent" |
497 | << "singleskip" |
498 | << "skip" |
499 | << "skipcleanup" |
500 | << "skipinit" |
501 | << "skipinitdata" |
502 | << "sleep" |
503 | << "strcmp" |
504 | << "subtest" |
505 | << "testlib" |
506 | << "tuplediagnostics" |
507 | << "verbose1" |
508 | << "verbose2" |
509 | #ifndef QT_NO_EXCEPTIONS |
510 | // this test will test nothing if the exceptions are disabled |
511 | << "verifyexceptionthrown" |
512 | #endif //!QT_NO_EXCEPTIONS |
513 | << "warnings" |
514 | << "watchdog" |
515 | << "xunit" |
516 | ; |
517 | |
518 | // These tests are affected by timing and whether the CPU tick counter |
519 | // is monotonically increasing. They won't work on some machines so |
520 | // leave them off by default. Feel free to enable them for your own |
521 | // testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment |
522 | // variable to something non-empty. |
523 | if (!qgetenv(varName: "QTEST_ENABLE_EXTRA_SELFTESTS" ).isEmpty()) |
524 | tests << "benchlibtickcounter" |
525 | << "benchlibwalltime" |
526 | ; |
527 | |
528 | foreach (LoggerSet const& loggerSet, allLoggerSets()) { |
529 | QStringList loggers = loggerSet.loggers; |
530 | |
531 | foreach (QString const& subtest, tests) { |
532 | // These tests don't work right unless logging plain text to |
533 | // standard output, either because they execute multiple test |
534 | // objects or because they internally supply arguments to |
535 | // themselves. |
536 | if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt" ) { |
537 | if (subtest == "differentexec" ) { |
538 | continue; |
539 | } |
540 | if (subtest == "multiexec" ) { |
541 | continue; |
542 | } |
543 | if (subtest == "qexecstringlist" ) { |
544 | continue; |
545 | } |
546 | if (subtest == "benchliboptions" ) { |
547 | continue; |
548 | } |
549 | if (subtest == "blacklisted" ) { |
550 | continue; |
551 | } |
552 | if (subtest == "printdatatags" ) { |
553 | continue; |
554 | } |
555 | if (subtest == "printdatatagswithglobaltags" ) { |
556 | continue; |
557 | } |
558 | if (subtest == "silent" ) { |
559 | continue; |
560 | } |
561 | // `crashes' will not output valid XML on platforms without a crash handler |
562 | if (subtest == "crashes" ) { |
563 | continue; |
564 | } |
565 | // this test prints out some floats in the testlog and the formatting is |
566 | // platform-specific and hard to predict. |
567 | if (subtest == "float" ) { |
568 | continue; |
569 | } |
570 | // these tests are quite slow, and running them for all the loggers significantly |
571 | // increases the overall test time. They do not really relate to logging, so it |
572 | // should be safe to run them just for the stdout loggers. |
573 | if (subtest == "benchlibcallgrind" || subtest == "sleep" ) { |
574 | continue; |
575 | } |
576 | } |
577 | if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains(s: "txt" ))) |
578 | continue; // XML only, do not mix txt and XML for encoding test. |
579 | |
580 | if (loggerSet.name.contains(s: "csv" ) && !subtest.startsWith(s: "benchlib" )) |
581 | continue; |
582 | |
583 | if (loggerSet.name.contains(s: "teamcity" ) && subtest.startsWith(s: "benchlib" )) |
584 | continue; // Skip benchmark for TeamCity logger |
585 | |
586 | // Keep in sync with generateTestData()'s crashers in generate_expected_output.py: |
587 | const bool crashes = subtest == QLatin1String("assert" ) || subtest == QLatin1String("exceptionthrow" ) |
588 | || subtest == QLatin1String("fetchbogus" ) || subtest == QLatin1String("crashedterminate" ) |
589 | || subtest == QLatin1String("faildatatype" ) || subtest == QLatin1String("failfetchtype" ) |
590 | || subtest == QLatin1String("crashes" ) || subtest == QLatin1String("silent" ) |
591 | || subtest == QLatin1String("blacklisted" ) || subtest == QLatin1String("watchdog" ); |
592 | QTest::newRow(qPrintable(QString("%1 %2" ).arg(subtest).arg(loggerSet.name))) |
593 | << subtest |
594 | << loggers |
595 | << loggerSet.arguments |
596 | << crashes |
597 | ; |
598 | } |
599 | } |
600 | } |
601 | |
602 | #if QT_CONFIG(process) |
603 | |
604 | static QProcessEnvironment processEnvironment() |
605 | { |
606 | static QProcessEnvironment result; |
607 | if (result.isEmpty()) { |
608 | const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment(); |
609 | const bool preserveLibPath = qEnvironmentVariableIsSet(varName: "QT_PRESERVE_TESTLIB_PATH" ); |
610 | foreach (const QString &key, systemEnvironment.keys()) { |
611 | const bool useVariable = key == QLatin1String("PATH" ) || key == QLatin1String("QT_QPA_PLATFORM" ) |
612 | #if defined(Q_OS_QNX) |
613 | || key == QLatin1String("GRAPHICS_ROOT" ) || key == QLatin1String("TZ" ) |
614 | #elif defined(Q_OS_UNIX) |
615 | || key == QLatin1String("HOME" ) || key == QLatin1String("USER" ) // Required for X11 on openSUSE |
616 | || key == QLatin1String("QEMU_SET_ENV" ) || key == QLatin1String("QEMU_LD_PREFIX" ) // Required for QEMU |
617 | # if !defined(Q_OS_MAC) |
618 | || key == QLatin1String("DISPLAY" ) || key == QLatin1String("XAUTHLOCALHOSTNAME" ) |
619 | || key.startsWith(s: QLatin1String("XDG_" )) || key == QLatin1String("XAUTHORITY" ) |
620 | # endif // !Q_OS_MAC |
621 | #endif // Q_OS_UNIX |
622 | #ifdef __COVERAGESCANNER__ |
623 | || key == QLatin1String("QT_TESTCOCOON_ACTIVE" ) |
624 | #endif |
625 | || ( preserveLibPath && (key == QLatin1String("QT_PLUGIN_PATH" ) |
626 | || key == QLatin1String("LD_LIBRARY_PATH" ))) |
627 | ; |
628 | if (useVariable) |
629 | result.insert(name: key, value: systemEnvironment.value(name: key)); |
630 | } |
631 | // Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/: |
632 | result.insert(QStringLiteral("QT_LOGGING_RULES" ), |
633 | // Must match generate_expected_output.py's main()'s value: |
634 | QStringLiteral("*.debug=true;qt.*=false" )); |
635 | } |
636 | return result; |
637 | } |
638 | |
639 | static inline QByteArray msgProcessError(const QString &binary, const QStringList &args, |
640 | const QProcessEnvironment &e, const QString &what) |
641 | { |
642 | QString result; |
643 | QTextStream(&result) <<"Error running " << binary << ' ' << args.join(sep: ' ') |
644 | << " with environment " << e.toStringList().join(sep: ' ') << ": " << what; |
645 | return result.toLocal8Bit(); |
646 | } |
647 | |
648 | void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes) |
649 | { |
650 | #if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX) |
651 | if (subdir == "benchlibcallgrind" ) { |
652 | QProcess checkProcess; |
653 | QStringList args; |
654 | args << QLatin1String("--version" ); |
655 | checkProcess.start(QLatin1String("valgrind" ), args); |
656 | if (!checkProcess.waitForFinished(-1)) |
657 | QSKIP(QString("Valgrind broken or not available. Not running %1 test!" ).arg(subdir).toLocal8Bit()); |
658 | } |
659 | #endif |
660 | |
661 | QProcess proc; |
662 | QProcessEnvironment environment = processEnvironment(); |
663 | // Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py: |
664 | if (crashes) { |
665 | environment.insert(name: "QTEST_DISABLE_CORE_DUMP" , value: "1" ); |
666 | environment.insert(name: "QTEST_DISABLE_STACK_DUMP" , value: "1" ); |
667 | if (subdir == QLatin1String("watchdog" )) |
668 | environment.insert(name: "QTEST_FUNCTION_TIMEOUT" , value: "100" ); |
669 | } |
670 | proc.setProcessEnvironment(environment); |
671 | const QString path = subdir + QLatin1Char('/') + subdir; |
672 | proc.start(program: path, arguments); |
673 | QVERIFY2(proc.waitForStarted(), msgProcessError(path, arguments, environment, QStringLiteral("Cannot start: " ) + proc.errorString())); |
674 | QVERIFY2(proc.waitForFinished(), msgProcessError(path, arguments, environment, QStringLiteral("Timed out: " ) + proc.errorString())); |
675 | if (!crashes) { |
676 | QVERIFY2(proc.exitStatus() == QProcess::NormalExit, |
677 | msgProcessError(path, arguments, environment, |
678 | QStringLiteral("Crashed: " ) + proc.errorString() |
679 | + QStringLiteral(": " ) + QString::fromLocal8Bit(proc.readAllStandardError()))); |
680 | } |
681 | |
682 | QList<QByteArray> actualOutputs; |
683 | for (int i = 0; i < loggers.count(); ++i) { |
684 | QString logFile = logName(logger: loggers[i]); |
685 | QByteArray out; |
686 | if (logFile.isEmpty()) { |
687 | out = proc.readAllStandardOutput(); |
688 | } else { |
689 | QFile file(logFile); |
690 | if (file.open(flags: QIODevice::ReadOnly)) |
691 | out = file.readAll(); |
692 | } |
693 | actualOutputs << out; |
694 | } |
695 | |
696 | const QByteArray err(proc.readAllStandardError()); |
697 | |
698 | // Some tests may output unpredictable strings to stderr, which we'll ignore. |
699 | // |
700 | // For instance, uncaught exceptions on Windows might say (depending on Windows |
701 | // version and JIT debugger settings): |
702 | // "This application has requested the Runtime to terminate it in an unusual way. |
703 | // Please contact the application's support team for more information." |
704 | // |
705 | // Also, tests which use valgrind may generate warnings if the toolchain is |
706 | // newer than the valgrind version, such that valgrind can't understand the |
707 | // debug information on the binary. |
708 | if (subdir != QLatin1String("exceptionthrow" ) |
709 | && subdir != QLatin1String("cmptest" ) // QImage comparison requires QGuiApplication |
710 | && subdir != QLatin1String("fetchbogus" ) |
711 | && subdir != QLatin1String("watchdog" ) |
712 | && subdir != QLatin1String("xunit" ) |
713 | #ifdef Q_CC_MINGW |
714 | && subdir != QLatin1String("blacklisted" ) // calls qFatal() |
715 | && subdir != QLatin1String("silent" ) // calls qFatal() |
716 | #endif |
717 | #ifdef Q_OS_LINUX |
718 | // QEMU outputs to stderr about uncaught signals |
719 | && !(EmulationDetector::isRunningArmOnX86() && |
720 | (subdir == QLatin1String("assert" ) |
721 | || subdir == QLatin1String("blacklisted" ) |
722 | || subdir == QLatin1String("crashes" ) |
723 | || subdir == QLatin1String("faildatatype" ) |
724 | || subdir == QLatin1String("failfetchtype" ) |
725 | || subdir == QLatin1String("silent" ) |
726 | ) |
727 | ) |
728 | #endif |
729 | && subdir != QLatin1String("benchlibcallgrind" )) |
730 | QVERIFY2(err.isEmpty(), err.constData()); |
731 | |
732 | for (int n = 0; n < loggers.count(); ++n) { |
733 | QString logger = loggers[n]; |
734 | if (n == 0 && subdir == QLatin1String("crashes" )) { |
735 | QByteArray &actual = actualOutputs[0]; |
736 | #ifndef Q_OS_WIN |
737 | // Remove digits of times to match the expected file. |
738 | const QByteArray timePattern("Function time:" ); |
739 | int timePos = actual.indexOf(a: timePattern); |
740 | if (timePos >= 0) { |
741 | timePos += timePattern.size(); |
742 | const int nextLinePos = actual.indexOf(c: '\n', from: timePos); |
743 | for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) { |
744 | if (actual.at(i: c) >= '0' && actual.at(i: c) <= '9') |
745 | actual.remove(index: c, len: 1); |
746 | } |
747 | } |
748 | #else // !Q_OS_WIN |
749 | // Remove stack trace which is output to stdout. |
750 | const int exceptionLogStart = actual.indexOf("A crash occurred in " ); |
751 | if (exceptionLogStart >= 0) |
752 | actual.truncate(exceptionLogStart); |
753 | #endif // Q_OS_WIN |
754 | } |
755 | |
756 | QList<QByteArray> res = splitLines(ba: actualOutputs[n]); |
757 | QString errorMessage; |
758 | QString expectedFileName = expectedFileNameFromTest(subdir, logger); |
759 | QByteArrayList exp = expectedResult(fileName: expectedFileName); |
760 | if (!exp.isEmpty()) { |
761 | if (!compareOutput(logger, subdir, rawOutput: actualOutputs[n], actual: res, expected: exp, errorMessage: &errorMessage)) { |
762 | errorMessage.prepend(s: QLatin1Char('"') + logger + QLatin1String("\", " ) |
763 | + expectedFileName + QLatin1Char(' ')); |
764 | errorMessage += QLatin1String("\nActual:\n" ) + QLatin1String(actualOutputs[n]); |
765 | const QByteArray diff = runDiff(expected: exp, actual: res); |
766 | if (!diff.isEmpty()) |
767 | errorMessage += QLatin1String("\nDiff:\n" ) + QLatin1String(diff); |
768 | QFAIL(qPrintable(errorMessage)); |
769 | } |
770 | } else { |
771 | // For the "crashes" and other tests, there are multiple versions of the |
772 | // expected output. Loop until a matching one is found. |
773 | bool ok = false; |
774 | for (int i = 1; !ok; ++i) { |
775 | expectedFileName = expectedFileNameFromTest(subdir: subdir + QLatin1Char('_') + QString::number(i), logger); |
776 | const QByteArrayList exp = expectedResult(fileName: expectedFileName); |
777 | if (exp.isEmpty()) |
778 | break; |
779 | QString errorMessage2; |
780 | ok = compareOutput(logger, subdir, rawOutput: actualOutputs[n], actual: res, expected: exp, errorMessage: &errorMessage2); |
781 | if (!ok) |
782 | errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": " ) + errorMessage2; |
783 | } |
784 | if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output. |
785 | errorMessage.prepend(s: QLatin1String("Cannot find a matching file for " ) + subdir); |
786 | errorMessage += QLatin1String("\nActual:\n" ); |
787 | QDebug(&errorMessage) << actualOutputs[n]; |
788 | QFAIL(qPrintable(errorMessage)); |
789 | } |
790 | } |
791 | } |
792 | } |
793 | |
794 | static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]" ); } |
795 | static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@" ); } |
796 | |
797 | bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir, |
798 | const QByteArray &rawOutput, const QByteArrayList &actual, |
799 | const QByteArrayList &expected, |
800 | QString *errorMessage) const |
801 | { |
802 | |
803 | if (actual.size() != expected.size()) { |
804 | *errorMessage = QString::fromLatin1(str: "Mismatch in line count. Expected %1 but got %2." ) |
805 | .arg(a: expected.size()).arg(a: actual.size()); |
806 | return false; |
807 | } |
808 | |
809 | // For xml output formats, verify that the log is valid XML. |
810 | if (logger.endsWith(s: QLatin1String("xml" )) && !checkXml(logger, rawOutput, errorMessage)) |
811 | return false; |
812 | |
813 | // Verify that the actual output is an acceptable match for the |
814 | // expected output. |
815 | |
816 | const QString qtVersion = QLatin1String(QT_VERSION_STR); |
817 | bool benchmark = false; |
818 | for (int i = 0, size = actual.size(); i < size; ++i) { |
819 | const QByteArray &actualLineBA = actual.at(i); |
820 | // the __FILE__ __LINE__ output is compiler dependent, skip it |
821 | if (actualLineBA.startsWith(c: " Loc: [" ) && actualLineBA.endsWith(c: ")]" )) |
822 | continue; |
823 | if (actualLineBA.endsWith(c: " : failure location" )) |
824 | continue; |
825 | if (actualLineBA.endsWith(c: " : message location" )) |
826 | continue; |
827 | |
828 | if (actualLineBA.startsWith(c: "Config: Using QtTest library" ) // Text build string |
829 | || actualLineBA.startsWith(c: " <QtBuild" ) // XML, Light XML build string |
830 | || (actualLineBA.startsWith(c: " <property value=" ) && actualLineBA.endsWith(c: "name=\"QtBuild\"/>" ))) { // XUNIT-XML build string |
831 | continue; |
832 | } |
833 | |
834 | QString actualLine = QString::fromLatin1(str: actualLineBA); |
835 | QString expectedLine = QString::fromLatin1(str: expected.at(i)); |
836 | expectedLine.replace(before: qtVersionPlaceHolder(), after: qtVersion); |
837 | |
838 | // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity |
839 | if (logger.endsWith(s: QLatin1String("teamcity" ))) { |
840 | static QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]" ); |
841 | actualLine.replace(re: teamcityLocRegExp, after: teamCityLocation()); |
842 | expectedLine.replace(re: teamcityLocRegExp, after: teamCityLocation()); |
843 | } |
844 | |
845 | if (logger.endsWith(s: QLatin1String("tap" ))) { |
846 | if (expectedLine.contains(s: QLatin1String("at:" )) |
847 | || expectedLine.contains(s: QLatin1String("file:" )) |
848 | || expectedLine.contains(s: QLatin1String("line:" ))) |
849 | actualLine = expectedLine; |
850 | } |
851 | |
852 | if (!compareLine(logger, subdir, benchmark, actualLine, |
853 | expLine: expectedLine, errorMessage)) { |
854 | errorMessage->prepend(s: QLatin1String("Line " ) + QString::number(i + 1) |
855 | + QLatin1String(": " )); |
856 | return false; |
857 | } |
858 | |
859 | benchmark = actualLineBA.startsWith(c: "RESULT : " ); |
860 | } |
861 | return true; |
862 | } |
863 | |
864 | bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, |
865 | bool benchmark, |
866 | const QString &actualLine, const QString &expectedLine, |
867 | QString *errorMessage) const |
868 | { |
869 | if (actualLine == expectedLine) |
870 | return true; |
871 | |
872 | if ((subdir == QLatin1String("assert" ) |
873 | || subdir == QLatin1String("faildatatype" ) || subdir == QLatin1String("failfetchtype" )) |
874 | && actualLine.contains(s: QLatin1String("ASSERT: " )) |
875 | && expectedLine.contains(s: QLatin1String("ASSERT: " ))) { |
876 | // Q_ASSERT uses __FILE__, the exact contents of which are |
877 | // undefined. If have we something that looks like a Q_ASSERT and we |
878 | // were expecting to see a Q_ASSERT, we'll skip the line. |
879 | return true; |
880 | } |
881 | |
882 | if (expectedLine.startsWith(s: QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce" ))) { |
883 | // On some platforms we compile without RTTI, and as a result we never throw an exception |
884 | if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()" )) { |
885 | *errorMessage = QString::fromLatin1(str: "'%1' != 'tst_Exception::throwException()'" ).arg(a: actualLine); |
886 | return false; |
887 | } |
888 | return true; |
889 | } |
890 | |
891 | if (benchmark || actualLine.startsWith(s: QLatin1String("<BenchmarkResult" )) |
892 | || (logger == QLatin1String("csv" ) && actualLine.startsWith(c: QLatin1Char('"')))) { |
893 | // Don't do a literal comparison for benchmark results, since |
894 | // results have some natural variance. |
895 | QString error; |
896 | BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error); |
897 | if (!error.isEmpty()) { |
898 | *errorMessage = QString::fromLatin1(str: "Actual line didn't parse as benchmark result: %1\nLine: %2" ).arg(args&: error, args: actualLine); |
899 | return false; |
900 | } |
901 | BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error); |
902 | if (!error.isEmpty()) { |
903 | *errorMessage = QString::fromLatin1(str: "Expected line didn't parse as benchmark result: %1\nLine: %2" ).arg(args&: error, args: expectedLine); |
904 | return false; |
905 | } |
906 | return compareBenchmarkResult(r1: actualResult, r2: expectedResult, errorMessage); |
907 | } |
908 | |
909 | if (actualLine.startsWith(s: QLatin1String(" <Duration msecs=" )) |
910 | || actualLine.startsWith(s: QLatin1String("<Duration msecs=" ))) { |
911 | static QRegularExpression durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>" ); |
912 | QRegularExpressionMatch match = durationRegExp.match(subject: actualLine); |
913 | if (match.hasMatch()) |
914 | return true; |
915 | *errorMessage = QString::fromLatin1(str: "Invalid Duration tag: '%1'" ).arg(a: actualLine); |
916 | return false; |
917 | } |
918 | |
919 | if (actualLine.startsWith(s: QLatin1String("Totals:" )) && expectedLine.startsWith(s: QLatin1String("Totals:" ))) |
920 | return true; |
921 | |
922 | const QLatin1String pointerPlaceholder("_POINTER_" ); |
923 | if (expectedLine.contains(s: pointerPlaceholder) |
924 | && (expectedLine.contains(s: QLatin1String("Signal: " )) |
925 | || expectedLine.contains(s: QLatin1String("Slot: " )))) { |
926 | QString actual = actualLine; |
927 | // We don't care about the pointer of the object to whom the signal belongs, so we |
928 | // replace it with _POINTER_, e.g.: |
929 | // Signal: SignalSlotClass(7ffd72245410) signalWithoutParameters () |
930 | // Signal: QThread(7ffd72245410) started () |
931 | // After this instance pointer we may have further pointers and |
932 | // references (with an @ prefix) as parameters of the signal or |
933 | // slot being invoked. |
934 | // Signal: SignalSlotClass(_POINTER_) qStringRefSignal ((QString&)@55f5fbb8dd40) |
935 | actual.replace(re: QRegularExpression("\\b[a-f0-9]{8,}\\b" ), after: pointerPlaceholder); |
936 | // Also change QEventDispatcher{Glib,Win32,etc.} to QEventDispatcherPlatform |
937 | actual.replace(re: QRegularExpression("\\b(QEventDispatcher)\\w+\\b" ), after: QLatin1String("\\1Platform" )); |
938 | if (actual != expectedLine) { |
939 | *errorMessage = msgMismatch(actual, expected: expectedLine); |
940 | return false; |
941 | } |
942 | return true; |
943 | } |
944 | |
945 | if (EmulationDetector::isRunningArmOnX86() && subdir == QLatin1String("float" )) { |
946 | // QEMU cheats at qfloat16, so outputs it as if it were a float. |
947 | if (actualLine.endsWith(s: QLatin1String("Actual (operandLeft) : 0.001" )) |
948 | && expectedLine.endsWith(s: QLatin1String("Actual (operandLeft) : 0.000999" ))) { |
949 | return true; |
950 | } |
951 | } |
952 | |
953 | *errorMessage = msgMismatch(actual: actualLine, expected: expectedLine); |
954 | return false; |
955 | } |
956 | |
957 | bool tst_Selftests::checkXml(const QString &logger, QByteArray xml, |
958 | QString *errorMessage) const |
959 | { |
960 | // lightxml intentionally skips the root element, which technically makes it |
961 | // not valid XML. |
962 | // We'll add that ourselves for the purpose of validation. |
963 | if (logger.endsWith(s: QLatin1String("lightxml" ))) { |
964 | xml.prepend(s: "<root>" ); |
965 | xml.append(s: "</root>" ); |
966 | } |
967 | |
968 | QXmlStreamReader reader(xml); |
969 | while (!reader.atEnd()) |
970 | reader.readNext(); |
971 | |
972 | if (reader.hasError()) { |
973 | const int lineNumber = int(reader.lineNumber()); |
974 | const QByteArray line = xml.split(sep: '\n').value(i: lineNumber - 1); |
975 | *errorMessage = QString::fromLatin1(str: "line %1, col %2 '%3': %4" ) |
976 | .arg(a: lineNumber).arg(a: reader.columnNumber()) |
977 | .arg(args: QString::fromLatin1(str: line), args: reader.errorString()); |
978 | return false; |
979 | } |
980 | return true; |
981 | } |
982 | |
983 | #endif // QT_CONFIG(process) |
984 | |
985 | void tst_Selftests::runSubTest() |
986 | { |
987 | #if !QT_CONFIG(process) |
988 | QSKIP("This test requires QProcess support" ); |
989 | #else |
990 | QFETCH(QString, subdir); |
991 | QFETCH(QStringList, loggers); |
992 | QFETCH(QStringList, arguments); |
993 | QFETCH(bool, crashes); |
994 | |
995 | doRunSubTest(subdir, loggers, arguments, crashes); |
996 | #endif // QT_CONFIG(process) |
997 | } |
998 | |
999 | // attribute must contain =" |
1000 | QString (const QString &line, const char *attribute) |
1001 | { |
1002 | int index = line.indexOf(s: attribute); |
1003 | if (index == -1) |
1004 | return QString(); |
1005 | const int attributeLength = int(strlen(s: attribute)); |
1006 | const int end = line.indexOf(c: '"', from: index + attributeLength); |
1007 | if (end == -1) |
1008 | return QString(); |
1009 | |
1010 | const QString result = line.mid(position: index + attributeLength, n: end - index - attributeLength); |
1011 | if (result.isEmpty()) |
1012 | return "" ; // ensure empty but not null |
1013 | return result; |
1014 | } |
1015 | |
1016 | // Parse line into the BenchmarkResult it represents. |
1017 | BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) |
1018 | { |
1019 | if (error) *error = QString(); |
1020 | BenchmarkResult out; |
1021 | |
1022 | QString remaining = line.trimmed(); |
1023 | |
1024 | if (remaining.isEmpty()) { |
1025 | if (error) *error = "Line is empty" ; |
1026 | return out; |
1027 | } |
1028 | |
1029 | if (line.startsWith(s: "<BenchmarkResult " )) { |
1030 | // XML result |
1031 | // format: |
1032 | // <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" /> |
1033 | if (!line.endsWith(s: "/>" )) { |
1034 | if (error) *error = "unterminated XML" ; |
1035 | return out; |
1036 | } |
1037 | |
1038 | QString unit = extractXmlAttribute(line, attribute: " metric=\"" ); |
1039 | QString sTotal = extractXmlAttribute(line, attribute: " value=\"" ); |
1040 | QString sIterations = extractXmlAttribute(line, attribute: " iterations=\"" ); |
1041 | if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) { |
1042 | if (error) *error = "XML snippet did not contain all required values" ; |
1043 | return out; |
1044 | } |
1045 | |
1046 | bool ok; |
1047 | double total = sTotal.toDouble(ok: &ok); |
1048 | if (!ok) { |
1049 | if (error) *error = sTotal + " is not a valid number" ; |
1050 | return out; |
1051 | } |
1052 | double iterations = sIterations.toDouble(ok: &ok); |
1053 | if (!ok) { |
1054 | if (error) *error = sIterations + " is not a valid number" ; |
1055 | return out; |
1056 | } |
1057 | |
1058 | out.unit = unit; |
1059 | out.total = total; |
1060 | out.iterations = iterations; |
1061 | return out; |
1062 | } |
1063 | |
1064 | if (line.startsWith(c: '"')) { |
1065 | // CSV result |
1066 | // format: |
1067 | // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations |
1068 | QStringList split = line.split(sep: ','); |
1069 | if (split.count() != 6) { |
1070 | if (error) *error = QString("Wrong number of columns (%1)" ).arg(a: split.count()); |
1071 | return out; |
1072 | } |
1073 | |
1074 | bool ok; |
1075 | double total = split.at(i: 4).toDouble(ok: &ok); |
1076 | if (!ok) { |
1077 | if (error) *error = split.at(i: 4) + " is not a valid number" ; |
1078 | return out; |
1079 | } |
1080 | double iterations = split.at(i: 5).toDouble(ok: &ok); |
1081 | if (!ok) { |
1082 | if (error) *error = split.at(i: 5) + " is not a valid number" ; |
1083 | return out; |
1084 | } |
1085 | |
1086 | out.unit = split.at(i: 2); |
1087 | out.total = total; |
1088 | out.iterations = iterations; |
1089 | return out; |
1090 | } |
1091 | |
1092 | // Text result |
1093 | // This code avoids using a QRegExp because QRegExp might be broken. |
1094 | // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1) |
1095 | |
1096 | QString sFirstNumber; |
1097 | while (!remaining.isEmpty() && !remaining.at(i: 0).isSpace()) { |
1098 | sFirstNumber += remaining.at(i: 0); |
1099 | remaining.remove(i: 0,len: 1); |
1100 | } |
1101 | remaining = remaining.trimmed(); |
1102 | |
1103 | // 4,000 -> 4000 |
1104 | sFirstNumber.remove(c: ','); |
1105 | |
1106 | // Should now be parseable as floating point |
1107 | bool ok; |
1108 | double firstNumber = sFirstNumber.toDouble(ok: &ok); |
1109 | if (!ok) { |
1110 | if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number" ; |
1111 | return out; |
1112 | } |
1113 | |
1114 | // Remaining: msec per iteration (total: 4000, iterations: 1) |
1115 | static const char periterbit[] = " per iteration (total: " ; |
1116 | QString unit; |
1117 | while (!remaining.startsWith(s: periterbit) && !remaining.isEmpty()) { |
1118 | unit += remaining.at(i: 0); |
1119 | remaining.remove(i: 0,len: 1); |
1120 | } |
1121 | if (remaining.isEmpty()) { |
1122 | if (error) *error = "Could not find pattern: '<unit> per iteration (total: '" ; |
1123 | return out; |
1124 | } |
1125 | |
1126 | remaining = remaining.mid(position: sizeof(periterbit)-1); |
1127 | |
1128 | // Remaining: 4,000, iterations: 1) |
1129 | static const char itersbit[] = ", iterations: " ; |
1130 | QString sTotal; |
1131 | while (!remaining.startsWith(s: itersbit) && !remaining.isEmpty()) { |
1132 | sTotal += remaining.at(i: 0); |
1133 | remaining.remove(i: 0,len: 1); |
1134 | } |
1135 | if (remaining.isEmpty()) { |
1136 | if (error) *error = "Could not find pattern: '<number>, iterations: '" ; |
1137 | return out; |
1138 | } |
1139 | |
1140 | remaining = remaining.mid(position: sizeof(itersbit)-1); |
1141 | |
1142 | // 4,000 -> 4000 |
1143 | sTotal.remove(c: ','); |
1144 | |
1145 | double total = sTotal.toDouble(ok: &ok); |
1146 | if (!ok) { |
1147 | if (error) *error = sTotal + " (total) is not a valid number" ; |
1148 | return out; |
1149 | } |
1150 | |
1151 | // Remaining: 1) |
1152 | QString sIters; |
1153 | while (remaining != QLatin1String(")" ) && !remaining.isEmpty()) { |
1154 | sIters += remaining.at(i: 0); |
1155 | remaining.remove(i: 0,len: 1); |
1156 | } |
1157 | if (remaining.isEmpty()) { |
1158 | if (error) *error = "Could not find pattern: '<num>)'" ; |
1159 | return out; |
1160 | } |
1161 | qint64 iters = sIters.toLongLong(ok: &ok); |
1162 | if (!ok) { |
1163 | if (error) *error = sIters + " (iterations) is not a valid integer" ; |
1164 | return out; |
1165 | } |
1166 | |
1167 | double calcFirstNumber = double(total)/double(iters); |
1168 | if (!qFuzzyCompare(p1: firstNumber, p2: calcFirstNumber)) { |
1169 | if (error) *error = QString("total/iters is %1, but benchlib output result as %2" ).arg(a: calcFirstNumber).arg(a: firstNumber); |
1170 | return out; |
1171 | } |
1172 | |
1173 | out.total = total; |
1174 | out.unit = unit; |
1175 | out.iterations = iters; |
1176 | return out; |
1177 | } |
1178 | |
1179 | void tst_Selftests::cleanup() |
1180 | { |
1181 | QFETCH(QStringList, loggers); |
1182 | |
1183 | // Remove the test output files |
1184 | for (int i = 0; i < loggers.count(); ++i) { |
1185 | QString logFileName = logName(logger: loggers[i]); |
1186 | if (!logFileName.isEmpty()) { |
1187 | QFile logFile(logFileName); |
1188 | if (logFile.exists()) |
1189 | QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: " ).arg(logFileName, logFile.errorString()))); |
1190 | } |
1191 | } |
1192 | } |
1193 | |
1194 | QTEST_MAIN(tst_Selftests) |
1195 | |
1196 | #include "tst_selftests.moc" |
1197 | |