1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/qtestassert.h>
5
6#include <QtTest/private/qtestlog_p.h>
7#include <QtTest/private/qtestresult_p.h>
8#include <QtTest/private/qabstracttestlogger_p.h>
9#include <QtTest/private/qplaintestlogger_p.h>
10#include <QtTest/private/qcsvbenchmarklogger_p.h>
11#include <QtTest/private/qjunittestlogger_p.h>
12#include <QtTest/private/qxmltestlogger_p.h>
13#include <QtTest/private/qteamcitylogger_p.h>
14#include <QtTest/private/qtaptestlogger_p.h>
15#if defined(HAVE_XCTEST)
16#include <QtTest/private/qxctestlogger_p.h>
17#endif
18
19#if defined(Q_OS_DARWIN)
20#include <QtTest/private/qappletestlogger_p.h>
21#endif
22
23#include <QtCore/qatomic.h>
24#include <QtCore/qbytearray.h>
25#include <QtCore/qelapsedtimer.h>
26#include <QtCore/qlist.h>
27#include <QtCore/qmutex.h>
28#include <QtCore/qvariant.h>
29#if QT_CONFIG(regularexpression)
30#include <QtCore/QRegularExpression>
31#endif
32
33#include <stdlib.h>
34#include <string.h>
35#include <limits.h>
36#include <vector>
37
38#include <vector>
39#include <memory>
40
41QT_BEGIN_NAMESPACE
42
43using namespace Qt::StringLiterals;
44
45static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
46{
47#ifdef __COVERAGESCANNER__
48# if QT_CONFIG(testlib_selfcover)
49 __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
50 QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
51# else
52 if (!installedTestCoverage)
53 return;
54 // install again to make sure the filename is correct.
55 // without this, a plugin or similar may have changed the filename.
56 __coveragescanner_install(appname);
57 __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
58 __coveragescanner_save();
59 __coveragescanner_testname("");
60 __coveragescanner_clear();
61 unsetenv("QT_TESTCOCOON_ACTIVE");
62# endif // testlib_selfcover
63#else
64 Q_UNUSED(appname);
65 Q_UNUSED(testfailed);
66 Q_UNUSED(installedTestCoverage);
67#endif
68}
69
70Q_CONSTINIT static QElapsedTimer elapsedFunctionTime;
71Q_CONSTINIT static QElapsedTimer elapsedTotalTime;
72
73#define FOREACH_TEST_LOGGER for (const auto &logger : std::as_const(*QTest::loggers()))
74
75namespace QTest {
76
77 int fails = 0;
78 int passes = 0;
79 int skips = 0;
80 int blacklists = 0;
81 enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState;
82
83 struct IgnoreResultList
84 {
85 inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
86 : type(tp), pattern(patternIn) {}
87
88 static inline void clearList(IgnoreResultList *&list)
89 {
90 while (list) {
91 IgnoreResultList *current = list;
92 list = list->next;
93 delete current;
94 }
95 }
96
97 static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
98 {
99 QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
100
101 if (!list) {
102 list = item;
103 return;
104 }
105 IgnoreResultList *last = list;
106 for ( ; last->next; last = last->next) ;
107 last->next = item;
108 }
109
110 static bool stringsMatch(const QString &expected, const QString &actual)
111 {
112 if (expected == actual)
113 return true;
114
115 // ignore an optional whitespace at the end of str
116 // (the space was added automatically by ~QDebug() until Qt 5.3,
117 // so autotests still might expect it)
118 if (expected.endsWith(c: u' '))
119 return actual == QStringView{expected}.left(n: expected.size() - 1);
120
121 return false;
122 }
123
124 inline bool matches(QtMsgType tp, const QString &message) const
125 {
126 return tp == type
127 && (pattern.userType() == QMetaType::QString ?
128 stringsMatch(expected: pattern.toString(), actual: message) :
129#if QT_CONFIG(regularexpression)
130 pattern.toRegularExpression().match(subject: message).hasMatch());
131#else
132 false);
133#endif
134 }
135
136 QtMsgType type;
137 QVariant pattern;
138 IgnoreResultList *next = nullptr;
139 };
140
141 static IgnoreResultList *ignoreResultList = nullptr;
142 Q_CONSTINIT static QBasicMutex mutex;
143
144 static std::vector<QVariant> failOnWarningList;
145
146 Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
147
148 static int verbosity = 0;
149 static int maxWarnings = 2002;
150 static bool installedTestCoverage = true;
151
152 static QtMessageHandler oldMessageHandler;
153
154 static bool handleIgnoredMessage(QtMsgType type, const QString &message)
155 {
156 const QMutexLocker mutexLocker(&QTest::mutex);
157
158 if (!ignoreResultList)
159 return false;
160 IgnoreResultList *last = nullptr;
161 IgnoreResultList *list = ignoreResultList;
162 while (list) {
163 if (list->matches(tp: type, message)) {
164 // remove the item from the list
165 if (last)
166 last->next = list->next;
167 else
168 ignoreResultList = list->next;
169
170 delete list;
171 return true;
172 }
173
174 last = list;
175 list = list->next;
176 }
177 return false;
178 }
179
180 static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
181 {
182 // failOnWarning can be called multiple times per test function, so let
183 // each call cause a failure if required.
184 for (const auto &pattern : failOnWarningList) {
185 if (pattern.metaType() == QMetaType::fromType<QString>()) {
186 if (message != pattern.toString())
187 continue;
188 }
189#if QT_CONFIG(regularexpression)
190 else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
191 if (!message.contains(re: pattern.toRegularExpression()))
192 continue;
193 }
194#endif
195
196 const size_t maxMsgLen = 1024;
197 char msg[maxMsgLen] = {'\0'};
198 qsnprintf(str: msg, n: maxMsgLen, fmt: "Received a warning that resulted in a failure:\n%s",
199 qPrintable(message));
200 QTestResult::addFailure(message: msg, file: context.file, line: context.line);
201 return true;
202 }
203 return false;
204 }
205
206 static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
207 {
208 static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
209
210 if (!QTestLog::hasLoggers()) {
211 // if this goes wrong, something is seriously broken.
212 qInstallMessageHandler(oldMessageHandler);
213 QTEST_ASSERT(QTestLog::hasLoggers());
214 }
215
216 if (handleIgnoredMessage(type, message)) {
217 // the message is expected, so just swallow it.
218 return;
219 }
220
221 if (type == QtWarningMsg && handleFailOnWarning(context, message))
222 return;
223
224 if (type != QtFatalMsg) {
225 if (counter.loadRelaxed() <= 0)
226 return;
227
228 if (!counter.deref()) {
229 FOREACH_TEST_LOGGER {
230 logger->addMessage(type: QAbstractTestLogger::Warn,
231 QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
232 }
233 return;
234 }
235 }
236
237 FOREACH_TEST_LOGGER
238 logger->addMessage(type, context, message);
239
240 if (type == QtFatalMsg) {
241 /* Right now, we're inside the custom message handler and we're
242 * being qt_message_output in qglobal.cpp. After we return from
243 * this function, it will proceed with calling exit() and abort()
244 * and hence crash. Therefore, we call these logging functions such
245 * that we wrap up nicely, and in particular produce well-formed XML. */
246 QTestResult::addFailure(message: "Received a fatal error.", file: context.file, line: context.line);
247 QTestLog::leaveTestFunction();
248 QTestLog::stopLogging();
249 }
250 }
251}
252
253void QTestLog::enterTestFunction(const char* function)
254{
255 elapsedFunctionTime.restart();
256 if (printAvailableTags)
257 return;
258
259 QTEST_ASSERT(function);
260
261 FOREACH_TEST_LOGGER
262 logger->enterTestFunction(function);
263}
264
265void QTestLog::enterTestData(QTestData *data)
266{
267 QTEST_ASSERT(data);
268
269 FOREACH_TEST_LOGGER
270 logger->enterTestData(data);
271}
272
273int QTestLog::unhandledIgnoreMessages()
274{
275 const QMutexLocker mutexLocker(&QTest::mutex);
276 int i = 0;
277 QTest::IgnoreResultList *list = QTest::ignoreResultList;
278 while (list) {
279 ++i;
280 list = list->next;
281 }
282 return i;
283}
284
285void QTestLog::leaveTestFunction()
286{
287 if (printAvailableTags)
288 return;
289
290 FOREACH_TEST_LOGGER
291 logger->leaveTestFunction();
292}
293
294void QTestLog::printUnhandledIgnoreMessages()
295{
296 const QMutexLocker mutexLocker(&QTest::mutex);
297 QString message;
298 QTest::IgnoreResultList *list = QTest::ignoreResultList;
299 while (list) {
300 if (list->pattern.userType() == QMetaType::QString) {
301 message = "Did not receive message: \"%1\""_L1.arg(args: list->pattern.toString());
302 } else {
303#if QT_CONFIG(regularexpression)
304 message = "Did not receive any message matching: \"%1\""_L1.arg(
305 args: list->pattern.toRegularExpression().pattern());
306#endif
307 }
308 FOREACH_TEST_LOGGER
309 logger->addMessage(type: QAbstractTestLogger::Info, message);
310
311 list = list->next;
312 }
313}
314
315void QTestLog::clearIgnoreMessages()
316{
317 const QMutexLocker mutexLocker(&QTest::mutex);
318 QTest::IgnoreResultList::clearList(list&: QTest::ignoreResultList);
319}
320
321
322void QTestLog::clearFailOnWarnings()
323{
324 QTest::failOnWarningList.clear();
325}
326
327void QTestLog::clearCurrentTestState()
328{
329 QTest::currentTestState = QTest::Unresolved;
330}
331
332void QTestLog::addPass(const char *msg)
333{
334 if (printAvailableTags)
335 return;
336
337 QTEST_ASSERT(msg);
338 Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
339
340 ++QTest::passes;
341 QTest::currentTestState = QTest::Passed;
342
343 FOREACH_TEST_LOGGER
344 logger->addIncident(type: QAbstractTestLogger::Pass, description: msg);
345}
346
347void QTestLog::addFail(const char *msg, const char *file, int line)
348{
349 QTEST_ASSERT(msg);
350
351 if (QTest::currentTestState == QTest::Unresolved) {
352 ++QTest::fails;
353 } else {
354 // After an XPASS/Continue, or fail or skip in a function the test
355 // calls, we can subsequently fail.
356 Q_ASSERT(QTest::currentTestState == QTest::Failed
357 || QTest::currentTestState == QTest::Skipped);
358 }
359 // It is up to particular loggers to decide whether to report such
360 // subsequent failures; they may carry useful information.
361
362 QTest::currentTestState = QTest::Failed;
363 FOREACH_TEST_LOGGER
364 logger->addIncident(type: QAbstractTestLogger::Fail, description: msg, file, line);
365}
366
367void QTestLog::addXFail(const char *msg, const char *file, int line)
368{
369 QTEST_ASSERT(msg);
370
371 // Will be counted in addPass() if we get there.
372
373 FOREACH_TEST_LOGGER
374 logger->addIncident(type: QAbstractTestLogger::XFail, description: msg, file, line);
375}
376
377void QTestLog::addXPass(const char *msg, const char *file, int line)
378{
379 QTEST_ASSERT(msg);
380
381 if (QTest::currentTestState == QTest::Unresolved) {
382 ++QTest::fails;
383 } else {
384 // After an XPASS/Continue, we can subsequently XPASS again.
385 // Likewise after a fail or skip in a function called by the test.
386 Q_ASSERT(QTest::currentTestState == QTest::Failed
387 || QTest::currentTestState == QTest::Skipped);
388 }
389
390 QTest::currentTestState = QTest::Failed;
391 FOREACH_TEST_LOGGER
392 logger->addIncident(type: QAbstractTestLogger::XPass, description: msg, file, line);
393}
394
395void QTestLog::addBPass(const char *msg)
396{
397 QTEST_ASSERT(msg);
398 Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
399
400 ++QTest::blacklists; // Not passes ?
401 QTest::currentTestState = QTest::Suppressed;
402
403 FOREACH_TEST_LOGGER
404 logger->addIncident(type: QAbstractTestLogger::BlacklistedPass, description: msg);
405}
406
407void QTestLog::addBFail(const char *msg, const char *file, int line)
408{
409 QTEST_ASSERT(msg);
410
411 if (QTest::currentTestState == QTest::Unresolved) {
412 ++QTest::blacklists;
413 } else {
414 // After a BXPASS/Continue, we can subsequently fail.
415 // Likewise after a fail or skip in a function called by a test.
416 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
417 || QTest::currentTestState == QTest::Skipped);
418 }
419
420 QTest::currentTestState = QTest::Suppressed;
421 FOREACH_TEST_LOGGER
422 logger->addIncident(type: QAbstractTestLogger::BlacklistedFail, description: msg, file, line);
423}
424
425void QTestLog::addBXPass(const char *msg, const char *file, int line)
426{
427 QTEST_ASSERT(msg);
428
429 if (QTest::currentTestState == QTest::Unresolved) {
430 ++QTest::blacklists;
431 } else {
432 // After a BXPASS/Continue, we may BXPASS again.
433 // Likewise after a fail or skip in a function called by a test.
434 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
435 || QTest::currentTestState == QTest::Skipped);
436 }
437
438 QTest::currentTestState = QTest::Suppressed;
439 FOREACH_TEST_LOGGER
440 logger->addIncident(type: QAbstractTestLogger::BlacklistedXPass, description: msg, file, line);
441}
442
443void QTestLog::addBXFail(const char *msg, const char *file, int line)
444{
445 QTEST_ASSERT(msg);
446
447 // Will be counted in addBPass() if we get there.
448
449 FOREACH_TEST_LOGGER
450 logger->addIncident(type: QAbstractTestLogger::BlacklistedXFail, description: msg, file, line);
451}
452
453void QTestLog::addSkip(const char *msg, const char *file, int line)
454{
455 QTEST_ASSERT(msg);
456
457 if (QTest::currentTestState == QTest::Unresolved) {
458 ++QTest::skips;
459 QTest::currentTestState = QTest::Skipped;
460 } else {
461 // After an B?XPASS/Continue, we might subsequently skip.
462 // Likewise after a skip in a function called by a test.
463 Q_ASSERT(QTest::currentTestState == QTest::Suppressed
464 || QTest::currentTestState == QTest::Failed
465 || QTest::currentTestState == QTest::Skipped);
466 }
467 // It is up to particular loggers to decide whether to report such
468 // subsequent skips; they may carry useful information.
469
470 FOREACH_TEST_LOGGER
471 logger->addIncident(type: QAbstractTestLogger::Skip, description: msg, file, line);
472}
473
474void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results)
475{
476 FOREACH_TEST_LOGGER
477 logger->addBenchmarkResults(result: results);
478}
479
480void QTestLog::startLogging()
481{
482 elapsedTotalTime.start();
483 elapsedFunctionTime.start();
484 FOREACH_TEST_LOGGER
485 logger->startLogging();
486 QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
487}
488
489void QTestLog::stopLogging()
490{
491 qInstallMessageHandler(QTest::oldMessageHandler);
492 FOREACH_TEST_LOGGER {
493 logger->stopLogging();
494 }
495 QTest::loggers()->clear();
496 saveCoverageTool(appname: QTestResult::currentAppName(), testfailed: failCount() != 0, installedTestCoverage: QTestLog::installedTestCoverage());
497}
498
499void QTestLog::addLogger(LogMode mode, const char *filename)
500{
501 if (filename && strcmp(s1: filename, s2: "-") == 0)
502 filename = nullptr;
503
504 QAbstractTestLogger *logger = nullptr;
505 switch (mode) {
506 case QTestLog::Plain:
507 logger = new QPlainTestLogger(filename);
508 break;
509 case QTestLog::CSV:
510 logger = new QCsvBenchmarkLogger(filename);
511 break;
512 case QTestLog::XML:
513 logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
514 break;
515 case QTestLog::LightXML:
516 logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
517 break;
518 case QTestLog::JUnitXML:
519 logger = new QJUnitTestLogger(filename);
520 break;
521 case QTestLog::TeamCity:
522 logger = new QTeamCityLogger(filename);
523 break;
524 case QTestLog::TAP:
525 logger = new QTapTestLogger(filename);
526 break;
527#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
528 case QTestLog::Apple:
529 logger = new QAppleTestLogger;
530 break;
531#endif
532#if defined(HAVE_XCTEST)
533 case QTestLog::XCTest:
534 logger = new QXcodeTestLogger;
535 break;
536#endif
537 }
538
539 QTEST_ASSERT(logger);
540 addLogger(logger);
541}
542
543/*!
544 \internal
545
546 Adds a new logger to the set of loggers that will be used
547 to report incidents and messages during testing.
548
549 The function takes ownership of the logger.
550*/
551void QTestLog::addLogger(QAbstractTestLogger *logger)
552{
553 QTEST_ASSERT(logger);
554 QTest::loggers()->emplace_back(args&: logger);
555}
556
557bool QTestLog::hasLoggers()
558{
559 return !QTest::loggers()->empty();
560}
561
562bool QTestLog::loggerUsingStdout()
563{
564 FOREACH_TEST_LOGGER {
565 if (logger->isLoggingToStdout())
566 return true;
567 }
568
569 return false;
570}
571
572void QTestLog::warn(const char *msg, const char *file, int line)
573{
574 QTEST_ASSERT(msg);
575
576 FOREACH_TEST_LOGGER
577 logger->addMessage(type: QAbstractTestLogger::Warn, message: QString::fromUtf8(utf8: msg), file, line);
578}
579
580void QTestLog::info(const char *msg, const char *file, int line)
581{
582 QTEST_ASSERT(msg);
583
584 FOREACH_TEST_LOGGER
585 logger->addMessage(type: QAbstractTestLogger::Info, message: QString::fromUtf8(utf8: msg), file, line);
586}
587
588void QTestLog::setVerboseLevel(int level)
589{
590 QTest::verbosity = level;
591}
592
593int QTestLog::verboseLevel()
594{
595 return QTest::verbosity;
596}
597
598void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
599{
600 QTEST_ASSERT(msg);
601
602 const QMutexLocker mutexLocker(&QTest::mutex);
603 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QString::fromUtf8(utf8: msg));
604}
605
606#if QT_CONFIG(regularexpression)
607void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
608{
609 QTEST_ASSERT(expression.isValid());
610
611 const QMutexLocker mutexLocker(&QTest::mutex);
612 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QVariant(expression));
613}
614#endif // QT_CONFIG(regularexpression)
615
616void QTestLog::failOnWarning(const char *msg)
617{
618 QTest::failOnWarningList.push_back(x: QString::fromUtf8(utf8: msg));
619}
620
621#if QT_CONFIG(regularexpression)
622void QTestLog::failOnWarning(const QRegularExpression &expression)
623{
624 QTEST_ASSERT(expression.isValid());
625
626 QTest::failOnWarningList.push_back(x: QVariant::fromValue(value: expression));
627}
628#endif // QT_CONFIG(regularexpression)
629
630void QTestLog::setMaxWarnings(int m)
631{
632 QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
633}
634
635bool QTestLog::printAvailableTags = false;
636
637void QTestLog::setPrintAvailableTagsMode()
638{
639 printAvailableTags = true;
640}
641
642int QTestLog::passCount()
643{
644 return QTest::passes;
645}
646
647int QTestLog::failCount()
648{
649 return QTest::fails;
650}
651
652int QTestLog::skipCount()
653{
654 return QTest::skips;
655}
656
657int QTestLog::blacklistCount()
658{
659 return QTest::blacklists;
660}
661
662int QTestLog::totalCount()
663{
664 return passCount() + failCount() + skipCount() + blacklistCount();
665}
666
667void QTestLog::resetCounters()
668{
669 QTest::passes = 0;
670 QTest::fails = 0;
671 QTest::skips = 0;
672}
673
674void QTestLog::setInstalledTestCoverage(bool installed)
675{
676 QTest::installedTestCoverage = installed;
677}
678
679bool QTestLog::installedTestCoverage()
680{
681 return QTest::installedTestCoverage;
682}
683
684qint64 QTestLog::nsecsTotalTime()
685{
686 return elapsedTotalTime.nsecsElapsed();
687}
688
689qint64 QTestLog::nsecsFunctionTime()
690{
691 return elapsedFunctionTime.nsecsElapsed();
692}
693
694QT_END_NAMESPACE
695
696#include "moc_qtestlog_p.cpp"
697

source code of qtbase/src/testlib/qtestlog.cpp