1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/qtestassert.h>
5
6#include <QtTest/private/qtestlog_p.h>
7#include <QtTest/private/qtestresult_p.h>
8#include <QtTest/private/qabstracttestlogger_p.h>
9#include <QtTest/private/qplaintestlogger_p.h>
10#include <QtTest/private/qcsvbenchmarklogger_p.h>
11#include <QtTest/private/qjunittestlogger_p.h>
12#include <QtTest/private/qxmltestlogger_p.h>
13#include <QtTest/private/qteamcitylogger_p.h>
14#include <QtTest/private/qtaptestlogger_p.h>
15#if defined(HAVE_XCTEST)
16#include <QtTest/private/qxctestlogger_p.h>
17#endif
18
19#if defined(Q_OS_DARWIN)
20#include <QtTest/private/qappletestlogger_p.h>
21#endif
22
23#include <QtCore/qatomic.h>
24#include <QtCore/qbytearray.h>
25#include <QtCore/qelapsedtimer.h>
26#include <QtCore/qlist.h>
27#include <QtCore/qmutex.h>
28#include <QtCore/qvariant.h>
29#if QT_CONFIG(regularexpression)
30#include <QtCore/QRegularExpression>
31#endif
32
33#include <cstdio>
34
35#include <stdlib.h>
36#include <string.h>
37#include <limits.h>
38
39#include <vector>
40#include <memory>
41
42QT_BEGIN_NAMESPACE
43
44using namespace Qt::StringLiterals;
45
46static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
47{
48#ifdef __COVERAGESCANNER__
49# if QT_CONFIG(testlib_selfcover)
50 __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
51 QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
52# else
53 if (!installedTestCoverage)
54 return;
55 // install again to make sure the filename is correct.
56 // without this, a plugin or similar may have changed the filename.
57 __coveragescanner_install(appname);
58 __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
59 __coveragescanner_save();
60 __coveragescanner_testname("");
61 __coveragescanner_clear();
62 unsetenv("QT_TESTCOCOON_ACTIVE");
63# endif // testlib_selfcover
64#else
65 Q_UNUSED(appname);
66 Q_UNUSED(testfailed);
67 Q_UNUSED(installedTestCoverage);
68#endif
69}
70
71Q_CONSTINIT static QElapsedTimer elapsedFunctionTime;
72Q_CONSTINIT static QElapsedTimer elapsedTotalTime;
73
74#define FOREACH_TEST_LOGGER for (const auto &logger : std::as_const(*QTest::loggers()))
75
76namespace QTest {
77
78 int fails = 0;
79 int passes = 0;
80 int skips = 0;
81 int blacklists = 0;
82 enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState;
83
84 struct IgnoreResultList
85 {
86 inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
87 : type(tp), pattern(patternIn) {}
88
89 static inline void clearList(IgnoreResultList *&list)
90 {
91 while (list) {
92 IgnoreResultList *current = list;
93 list = list->next;
94 delete current;
95 }
96 }
97
98 static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
99 {
100 QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
101
102 if (!list) {
103 list = item;
104 return;
105 }
106 IgnoreResultList *last = list;
107 for ( ; last->next; last = last->next) ;
108 last->next = item;
109 }
110
111 static bool stringsMatch(const QString &expected, const QString &actual)
112 {
113 if (expected == actual)
114 return true;
115
116 // ignore an optional whitespace at the end of str
117 // (the space was added automatically by ~QDebug() until Qt 5.3,
118 // so autotests still might expect it)
119 if (expected.endsWith(c: u' '))
120 return actual == QStringView{expected}.left(n: expected.size() - 1);
121
122 return false;
123 }
124
125 inline bool matches(QtMsgType tp, const QString &message) const
126 {
127 return tp == type
128 && (pattern.userType() == QMetaType::QString ?
129 stringsMatch(expected: pattern.toString(), actual: message) :
130#if QT_CONFIG(regularexpression)
131 pattern.toRegularExpression().match(subject: message).hasMatch());
132#else
133 false);
134#endif
135 }
136
137 QtMsgType type;
138 QVariant pattern;
139 IgnoreResultList *next = nullptr;
140 };
141
142 static IgnoreResultList *ignoreResultList = nullptr;
143 Q_CONSTINIT static QBasicMutex mutex;
144
145 static std::vector<QVariant> failOnWarningList;
146
147 Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
148
149 static int verbosity = 0;
150 static int maxWarnings = 2002;
151 static bool installedTestCoverage = true;
152
153 static QtMessageHandler oldMessageHandler;
154
155 static bool handleIgnoredMessage(QtMsgType type, const QString &message)
156 {
157 const QMutexLocker mutexLocker(&QTest::mutex);
158
159 if (!ignoreResultList)
160 return false;
161 IgnoreResultList *last = nullptr;
162 IgnoreResultList *list = ignoreResultList;
163 while (list) {
164 if (list->matches(tp: type, message)) {
165 // remove the item from the list
166 if (last)
167 last->next = list->next;
168 else
169 ignoreResultList = list->next;
170
171 delete list;
172 return true;
173 }
174
175 last = list;
176 list = list->next;
177 }
178 return false;
179 }
180
181 static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
182 {
183 // failOnWarning can be called multiple times per test function, so let
184 // each call cause a failure if required.
185 for (const auto &pattern : failOnWarningList) {
186 if (pattern.metaType() == QMetaType::fromType<QString>()) {
187 if (message != pattern.toString())
188 continue;
189 }
190#if QT_CONFIG(regularexpression)
191 else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
192 if (!message.contains(re: pattern.toRegularExpression()))
193 continue;
194 }
195#endif
196
197 const size_t maxMsgLen = 1024;
198 char msg[maxMsgLen] = {'\0'};
199 std::snprintf(s: msg, maxlen: maxMsgLen, format: "Received a warning that resulted in a failure:\n%s",
200 qPrintable(message));
201 QTestResult::addFailure(message: msg, file: context.file, line: context.line);
202 return true;
203 }
204 return false;
205 }
206
207 static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
208 {
209 static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
210
211 if (!QTestLog::hasLoggers()) {
212 // if this goes wrong, something is seriously broken.
213 qInstallMessageHandler(oldMessageHandler);
214 QTEST_ASSERT(QTestLog::hasLoggers());
215 }
216
217 if (handleIgnoredMessage(type, message)) {
218 // the message is expected, so just swallow it.
219 return;
220 }
221
222 if (type == QtWarningMsg && handleFailOnWarning(context, message))
223 return;
224
225 if (type != QtFatalMsg) {
226 if (counter.loadRelaxed() <= 0)
227 return;
228
229 if (!counter.deref()) {
230 FOREACH_TEST_LOGGER {
231 logger->addMessage(type: QAbstractTestLogger::Warn,
232 QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
233 }
234 return;
235 }
236 }
237
238 FOREACH_TEST_LOGGER
239 logger->addMessage(type, context, message);
240
241 if (type == QtFatalMsg) {
242 /* Right now, we're inside the custom message handler and we're
243 * being qt_message_output in qglobal.cpp. After we return from
244 * this function, it will proceed with calling exit() and abort()
245 * and hence crash. Therefore, we call these logging functions such
246 * that we wrap up nicely, and in particular produce well-formed XML. */
247 QTestResult::addFailure(message: "Received a fatal error.", file: context.file, line: context.line);
248 QTestLog::leaveTestFunction();
249 QTestLog::stopLogging();
250 }
251 }
252}
253
254void QTestLog::enterTestFunction(const char* function)
255{
256 elapsedFunctionTime.restart();
257 if (printAvailableTags)
258 return;
259
260 QTEST_ASSERT(function);
261
262 FOREACH_TEST_LOGGER
263 logger->enterTestFunction(function);
264}
265
266void QTestLog::enterTestData(QTestData *data)
267{
268 QTEST_ASSERT(data);
269
270 FOREACH_TEST_LOGGER
271 logger->enterTestData(data);
272}
273
274int QTestLog::unhandledIgnoreMessages()
275{
276 const QMutexLocker mutexLocker(&QTest::mutex);
277 int i = 0;
278 QTest::IgnoreResultList *list = QTest::ignoreResultList;
279 while (list) {
280 ++i;
281 list = list->next;
282 }
283 return i;
284}
285
286void QTestLog::leaveTestFunction()
287{
288 if (printAvailableTags)
289 return;
290
291 FOREACH_TEST_LOGGER
292 logger->leaveTestFunction();
293}
294
295void QTestLog::printUnhandledIgnoreMessages()
296{
297 const QMutexLocker mutexLocker(&QTest::mutex);
298 QString message;
299 QTest::IgnoreResultList *list = QTest::ignoreResultList;
300 while (list) {
301 if (list->pattern.userType() == QMetaType::QString) {
302 message = "Did not receive message: \"%1\""_L1.arg(args: list->pattern.toString());
303 } else {
304#if QT_CONFIG(regularexpression)
305 message = "Did not receive any message matching: \"%1\""_L1.arg(
306 args: list->pattern.toRegularExpression().pattern());
307#endif
308 }
309 FOREACH_TEST_LOGGER
310 logger->addMessage(type: QAbstractTestLogger::Info, message);
311
312 list = list->next;
313 }
314}
315
316void QTestLog::clearIgnoreMessages()
317{
318 const QMutexLocker mutexLocker(&QTest::mutex);
319 QTest::IgnoreResultList::clearList(list&: QTest::ignoreResultList);
320}
321
322void QTestLog::clearFailOnWarnings()
323{
324 QTest::failOnWarningList.clear();
325}
326
327void QTestLog::clearCurrentTestState()
328{
329 clearIgnoreMessages();
330 clearFailOnWarnings();
331 QTest::currentTestState = QTest::Unresolved;
332}
333
334void QTestLog::addPass(const char *msg)
335{
336 if (printAvailableTags)
337 return;
338
339 QTEST_ASSERT(msg);
340 Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
341
342 ++QTest::passes;
343 QTest::currentTestState = QTest::Passed;
344
345 FOREACH_TEST_LOGGER
346 logger->addIncident(type: QAbstractTestLogger::Pass, description: msg);
347}
348
349void QTestLog::addFail(const char *msg, const char *file, int line)
350{
351 QTEST_ASSERT(msg);
352
353 if (QTest::currentTestState == QTest::Unresolved) {
354 ++QTest::fails;
355 } else {
356 // After an XPASS/Continue, or fail or skip in a function the test
357 // calls, we can subsequently fail.
358 Q_ASSERT(QTest::currentTestState == QTest::Failed
359 || QTest::currentTestState == QTest::Skipped);
360 }
361 // It is up to particular loggers to decide whether to report such
362 // subsequent failures; they may carry useful information.
363
364 QTest::currentTestState = QTest::Failed;
365 FOREACH_TEST_LOGGER
366 logger->addIncident(type: QAbstractTestLogger::Fail, description: msg, file, line);
367}
368
369void QTestLog::addXFail(const char *msg, const char *file, int line)
370{
371 QTEST_ASSERT(msg);
372
373 // Will be counted in addPass() if we get there.
374
375 FOREACH_TEST_LOGGER
376 logger->addIncident(type: QAbstractTestLogger::XFail, description: msg, file, line);
377}
378
379void QTestLog::addXPass(const char *msg, const char *file, int line)
380{
381 QTEST_ASSERT(msg);
382
383 if (QTest::currentTestState == QTest::Unresolved) {
384 ++QTest::fails;
385 } else {
386 // After an XPASS/Continue, we can subsequently XPASS again.
387 // Likewise after a fail or skip in a function called by the test.
388 Q_ASSERT(QTest::currentTestState == QTest::Failed
389 || QTest::currentTestState == QTest::Skipped);
390 }
391
392 QTest::currentTestState = QTest::Failed;
393 FOREACH_TEST_LOGGER
394 logger->addIncident(type: QAbstractTestLogger::XPass, description: msg, file, line);
395}
396
397void QTestLog::addBPass(const char *msg)
398{
399 QTEST_ASSERT(msg);
400 Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
401
402 ++QTest::blacklists; // Not passes ?
403 QTest::currentTestState = QTest::Suppressed;
404
405 FOREACH_TEST_LOGGER
406 logger->addIncident(type: QAbstractTestLogger::BlacklistedPass, description: msg);
407}
408
409void QTestLog::addBFail(const char *msg, const char *file, int line)
410{
411 QTEST_ASSERT(msg);
412
413 if (QTest::currentTestState == QTest::Unresolved) {
414 ++QTest::blacklists;
415 } else {
416 // After a BXPASS/Continue, we can subsequently fail.
417 // Likewise after a fail or skip in a function called by a test.
418 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
419 || QTest::currentTestState == QTest::Skipped);
420 }
421
422 QTest::currentTestState = QTest::Suppressed;
423 FOREACH_TEST_LOGGER
424 logger->addIncident(type: QAbstractTestLogger::BlacklistedFail, description: msg, file, line);
425}
426
427void QTestLog::addBXPass(const char *msg, const char *file, int line)
428{
429 QTEST_ASSERT(msg);
430
431 if (QTest::currentTestState == QTest::Unresolved) {
432 ++QTest::blacklists;
433 } else {
434 // After a BXPASS/Continue, we may BXPASS again.
435 // Likewise after a fail or skip in a function called by a test.
436 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
437 || QTest::currentTestState == QTest::Skipped);
438 }
439
440 QTest::currentTestState = QTest::Suppressed;
441 FOREACH_TEST_LOGGER
442 logger->addIncident(type: QAbstractTestLogger::BlacklistedXPass, description: msg, file, line);
443}
444
445void QTestLog::addBXFail(const char *msg, const char *file, int line)
446{
447 QTEST_ASSERT(msg);
448
449 // Will be counted in addBPass() if we get there.
450
451 FOREACH_TEST_LOGGER
452 logger->addIncident(type: QAbstractTestLogger::BlacklistedXFail, description: msg, file, line);
453}
454
455void QTestLog::addSkip(const char *msg, const char *file, int line)
456{
457 QTEST_ASSERT(msg);
458
459 if (QTest::currentTestState == QTest::Unresolved) {
460 ++QTest::skips;
461 QTest::currentTestState = QTest::Skipped;
462 } else {
463 // After an B?XPASS/Continue, we might subsequently skip.
464 // Likewise after a skip in a function called by a test.
465 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
466 || QTest::currentTestState == QTest::Failed
467 || QTest::currentTestState == QTest::Skipped);
468 }
469 // It is up to particular loggers to decide whether to report such
470 // subsequent skips; they may carry useful information.
471
472 FOREACH_TEST_LOGGER
473 logger->addIncident(type: QAbstractTestLogger::Skip, description: msg, file, line);
474}
475
476void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results)
477{
478 FOREACH_TEST_LOGGER
479 logger->addBenchmarkResults(result: results);
480}
481
482void QTestLog::startLogging()
483{
484 elapsedTotalTime.start();
485 elapsedFunctionTime.start();
486 FOREACH_TEST_LOGGER
487 logger->startLogging();
488 QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
489}
490
491void QTestLog::stopLogging()
492{
493 qInstallMessageHandler(QTest::oldMessageHandler);
494 FOREACH_TEST_LOGGER {
495 logger->stopLogging();
496 }
497 QTest::loggers()->clear();
498 saveCoverageTool(appname: QTestResult::currentAppName(), testfailed: failCount() != 0, installedTestCoverage: QTestLog::installedTestCoverage());
499}
500
501void QTestLog::addLogger(LogMode mode, const char *filename)
502{
503 if (filename && strcmp(s1: filename, s2: "-") == 0)
504 filename = nullptr;
505
506 QAbstractTestLogger *logger = nullptr;
507 switch (mode) {
508 case QTestLog::Plain:
509 logger = new QPlainTestLogger(filename);
510 break;
511 case QTestLog::CSV:
512 logger = new QCsvBenchmarkLogger(filename);
513 break;
514 case QTestLog::XML:
515 logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
516 break;
517 case QTestLog::LightXML:
518 logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
519 break;
520 case QTestLog::JUnitXML:
521 logger = new QJUnitTestLogger(filename);
522 break;
523 case QTestLog::TeamCity:
524 logger = new QTeamCityLogger(filename);
525 break;
526 case QTestLog::TAP:
527 logger = new QTapTestLogger(filename);
528 break;
529#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
530 case QTestLog::Apple:
531 logger = new QAppleTestLogger;
532 break;
533#endif
534#if defined(HAVE_XCTEST)
535 case QTestLog::XCTest:
536 logger = new QXcodeTestLogger;
537 break;
538#endif
539 }
540
541 QTEST_ASSERT(logger);
542 addLogger(logger);
543}
544
545/*!
546 \internal
547
548 Adds a new logger to the set of loggers that will be used
549 to report incidents and messages during testing.
550
551 The function takes ownership of the logger.
552*/
553void QTestLog::addLogger(QAbstractTestLogger *logger)
554{
555 QTEST_ASSERT(logger);
556 QTest::loggers()->emplace_back(args&: logger);
557}
558
559bool QTestLog::hasLoggers()
560{
561 return !QTest::loggers()->empty();
562}
563
564/*!
565 \internal
566
567 Returns true if all loggers support repeated test runs
568*/
569bool QTestLog::isRepeatSupported()
570{
571 FOREACH_TEST_LOGGER {
572 if (!logger->isRepeatSupported())
573 return false;
574 }
575
576 return true;
577}
578
579bool QTestLog::loggerUsingStdout()
580{
581 FOREACH_TEST_LOGGER {
582 if (logger->isLoggingToStdout())
583 return true;
584 }
585
586 return false;
587}
588
589void QTestLog::warn(const char *msg, const char *file, int line)
590{
591 QTEST_ASSERT(msg);
592
593 FOREACH_TEST_LOGGER
594 logger->addMessage(type: QAbstractTestLogger::Warn, message: QString::fromUtf8(utf8: msg), file, line);
595}
596
597void QTestLog::info(const char *msg, const char *file, int line)
598{
599 QTEST_ASSERT(msg);
600
601 FOREACH_TEST_LOGGER
602 logger->addMessage(type: QAbstractTestLogger::Info, message: QString::fromUtf8(utf8: msg), file, line);
603}
604
605void QTestLog::setVerboseLevel(int level)
606{
607 QTest::verbosity = level;
608}
609
610int QTestLog::verboseLevel()
611{
612 return QTest::verbosity;
613}
614
615void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
616{
617 QTEST_ASSERT(msg);
618
619 const QMutexLocker mutexLocker(&QTest::mutex);
620 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QString::fromUtf8(utf8: msg));
621}
622
623#if QT_CONFIG(regularexpression)
624void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
625{
626 QTEST_ASSERT(expression.isValid());
627
628 const QMutexLocker mutexLocker(&QTest::mutex);
629 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QVariant(expression));
630}
631#endif // QT_CONFIG(regularexpression)
632
633void QTestLog::failOnWarning()
634{
635 QTest::failOnWarningList.push_back(x: {});
636}
637
638void QTestLog::failOnWarning(const char *msg)
639{
640 QTest::failOnWarningList.push_back(x: QString::fromUtf8(utf8: msg));
641}
642
643#if QT_CONFIG(regularexpression)
644void QTestLog::failOnWarning(const QRegularExpression &expression)
645{
646 QTEST_ASSERT(expression.isValid());
647
648 QTest::failOnWarningList.push_back(x: QVariant::fromValue(value: expression));
649}
650#endif // QT_CONFIG(regularexpression)
651
652void QTestLog::setMaxWarnings(int m)
653{
654 QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
655}
656
657bool QTestLog::printAvailableTags = false;
658
659void QTestLog::setPrintAvailableTagsMode()
660{
661 printAvailableTags = true;
662}
663
664int QTestLog::passCount()
665{
666 return QTest::passes;
667}
668
669int QTestLog::failCount()
670{
671 return QTest::fails;
672}
673
674int QTestLog::skipCount()
675{
676 return QTest::skips;
677}
678
679int QTestLog::blacklistCount()
680{
681 return QTest::blacklists;
682}
683
684int QTestLog::totalCount()
685{
686 return passCount() + failCount() + skipCount() + blacklistCount();
687}
688
689void QTestLog::resetCounters()
690{
691 QTest::passes = 0;
692 QTest::fails = 0;
693 QTest::skips = 0;
694}
695
696void QTestLog::setInstalledTestCoverage(bool installed)
697{
698 QTest::installedTestCoverage = installed;
699}
700
701bool QTestLog::installedTestCoverage()
702{
703 return QTest::installedTestCoverage;
704}
705
706qint64 QTestLog::nsecsTotalTime()
707{
708 return elapsedTotalTime.nsecsElapsed();
709}
710
711qint64 QTestLog::nsecsFunctionTime()
712{
713 return elapsedFunctionTime.nsecsElapsed();
714}
715
716QT_END_NAMESPACE
717
718#include "moc_qtestlog_p.cpp"
719

Provided by KDAB

Privacy Policy
Learn to use CMake with our Intro Training
Find out more

source code of qtbase/src/testlib/qtestlog.cpp