1/****************************************************************************
2**
3** Copyright (C) 2020 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include <QtTest/qtestassert.h>
41
42#include <QtTest/private/qtestlog_p.h>
43#include <QtTest/private/qtestresult_p.h>
44#include <QtTest/private/qabstracttestlogger_p.h>
45#include <QtTest/private/qplaintestlogger_p.h>
46#include <QtTest/private/qcsvbenchmarklogger_p.h>
47#include <QtTest/private/qjunittestlogger_p.h>
48#include <QtTest/private/qxmltestlogger_p.h>
49#include <QtTest/private/qteamcitylogger_p.h>
50#include <QtTest/private/qtaptestlogger_p.h>
51#if defined(HAVE_XCTEST)
52#include <QtTest/private/qxctestlogger_p.h>
53#endif
54
55#if defined(Q_OS_DARWIN)
56#include <QtTest/private/qappletestlogger_p.h>
57#endif
58
59#include <QtCore/qatomic.h>
60#include <QtCore/qbytearray.h>
61#include <QtCore/QElapsedTimer>
62#include <QtCore/QVariant>
63#include <QtCore/qvector.h>
64#if QT_CONFIG(regularexpression)
65#include <QtCore/QRegularExpression>
66#endif
67
68#include <stdlib.h>
69#include <string.h>
70#include <limits.h>
71
72QT_BEGIN_NAMESPACE
73
74static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
75{
76#ifdef __COVERAGESCANNER__
77# if QT_CONFIG(testlib_selfcover)
78 __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
79 QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
80# else
81 if (!installedTestCoverage)
82 return;
83 // install again to make sure the filename is correct.
84 // without this, a plugin or similar may have changed the filename.
85 __coveragescanner_install(appname);
86 __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
87 __coveragescanner_save();
88 __coveragescanner_testname("");
89 __coveragescanner_clear();
90 unsetenv("QT_TESTCOCOON_ACTIVE");
91# endif // testlib_selfcover
92#else
93 Q_UNUSED(appname);
94 Q_UNUSED(testfailed);
95 Q_UNUSED(installedTestCoverage);
96#endif
97}
98
99static QElapsedTimer elapsedFunctionTime;
100static QElapsedTimer elapsedTotalTime;
101
102#define FOREACH_TEST_LOGGER for (QAbstractTestLogger *logger : *QTest::loggers())
103
104namespace QTest {
105
106 int fails = 0;
107 int passes = 0;
108 int skips = 0;
109 int blacklists = 0;
110
111 struct IgnoreResultList
112 {
113 inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
114 : type(tp), pattern(patternIn) {}
115
116 static inline void clearList(IgnoreResultList *&list)
117 {
118 while (list) {
119 IgnoreResultList *current = list;
120 list = list->next;
121 delete current;
122 }
123 }
124
125 static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
126 {
127 QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
128
129 if (!list) {
130 list = item;
131 return;
132 }
133 IgnoreResultList *last = list;
134 for ( ; last->next; last = last->next) ;
135 last->next = item;
136 }
137
138 static bool stringsMatch(const QString &expected, const QString &actual)
139 {
140 if (expected == actual)
141 return true;
142
143 // ignore an optional whitespace at the end of str
144 // (the space was added automatically by ~QDebug() until Qt 5.3,
145 // so autotests still might expect it)
146 if (expected.endsWith(c: QLatin1Char(' ')))
147 return actual == expected.leftRef(n: expected.length() - 1);
148
149 return false;
150 }
151
152 inline bool matches(QtMsgType tp, const QString &message) const
153 {
154 return tp == type
155 && (pattern.userType() == QMetaType::QString ?
156 stringsMatch(expected: pattern.toString(), actual: message) :
157#if QT_CONFIG(regularexpression)
158 pattern.toRegularExpression().match(subject: message).hasMatch());
159#else
160 false);
161#endif
162 }
163
164 QtMsgType type;
165 QVariant pattern;
166 IgnoreResultList *next = nullptr;
167 };
168
169 static IgnoreResultList *ignoreResultList = nullptr;
170
171 Q_GLOBAL_STATIC(QVector<QAbstractTestLogger *>, loggers)
172
173 static int verbosity = 0;
174 static int maxWarnings = 2002;
175 static bool installedTestCoverage = true;
176
177 static QtMessageHandler oldMessageHandler;
178
179 static bool handleIgnoredMessage(QtMsgType type, const QString &message)
180 {
181 if (!ignoreResultList)
182 return false;
183 IgnoreResultList *last = nullptr;
184 IgnoreResultList *list = ignoreResultList;
185 while (list) {
186 if (list->matches(tp: type, message)) {
187 // remove the item from the list
188 if (last)
189 last->next = list->next;
190 else if (list->next)
191 ignoreResultList = list->next;
192 else
193 ignoreResultList = nullptr;
194
195 delete list;
196 return true;
197 }
198
199 last = list;
200 list = list->next;
201 }
202 return false;
203 }
204
205 static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
206 {
207 static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
208
209 if (QTestLog::loggerCount() == 0) {
210 // if this goes wrong, something is seriously broken.
211 qInstallMessageHandler(oldMessageHandler);
212 QTEST_ASSERT(QTestLog::loggerCount() != 0);
213 }
214
215 if (handleIgnoredMessage(type, message)) {
216 // the message is expected, so just swallow it.
217 return;
218 }
219
220 if (type != QtFatalMsg) {
221 if (counter.loadRelaxed() <= 0)
222 return;
223
224 if (!counter.deref()) {
225 FOREACH_TEST_LOGGER {
226 logger->addMessage(type: QAbstractTestLogger::QSystem,
227 QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
228 }
229 return;
230 }
231 }
232
233 FOREACH_TEST_LOGGER
234 logger->addMessage(type, context, message);
235
236 if (type == QtFatalMsg) {
237 /* Right now, we're inside the custom message handler and we're
238 * being qt_message_output in qglobal.cpp. After we return from
239 * this function, it will proceed with calling exit() and abort()
240 * and hence crash. Therefore, we call these logging functions such
241 * that we wrap up nicely, and in particular produce well-formed XML. */
242 QTestResult::addFailure(message: "Received a fatal error.", file: "Unknown file", line: 0);
243 QTestLog::leaveTestFunction();
244 QTestLog::stopLogging();
245 }
246 }
247}
248
249void QTestLog::enterTestFunction(const char* function)
250{
251 elapsedFunctionTime.restart();
252 if (printAvailableTags)
253 return;
254
255 QTEST_ASSERT(function);
256
257 FOREACH_TEST_LOGGER
258 logger->enterTestFunction(function);
259}
260
261void QTestLog::enterTestData(QTestData *data)
262{
263 QTEST_ASSERT(data);
264
265 FOREACH_TEST_LOGGER
266 logger->enterTestData(data);
267}
268
269int QTestLog::unhandledIgnoreMessages()
270{
271 int i = 0;
272 QTest::IgnoreResultList *list = QTest::ignoreResultList;
273 while (list) {
274 ++i;
275 list = list->next;
276 }
277 return i;
278}
279
280void QTestLog::leaveTestFunction()
281{
282 if (printAvailableTags)
283 return;
284
285 FOREACH_TEST_LOGGER
286 logger->leaveTestFunction();
287}
288
289void QTestLog::printUnhandledIgnoreMessages()
290{
291 QString message;
292 QTest::IgnoreResultList *list = QTest::ignoreResultList;
293 while (list) {
294 if (list->pattern.userType() == QMetaType::QString) {
295 message = QStringLiteral("Did not receive message: \"") + list->pattern.toString() + QLatin1Char('"');
296 } else {
297#if QT_CONFIG(regularexpression)
298 message = QStringLiteral("Did not receive any message matching: \"") + list->pattern.toRegularExpression().pattern() + QLatin1Char('"');
299#endif
300 }
301 FOREACH_TEST_LOGGER
302 logger->addMessage(type: QAbstractTestLogger::Info, message);
303
304 list = list->next;
305 }
306}
307
308void QTestLog::clearIgnoreMessages()
309{
310 QTest::IgnoreResultList::clearList(list&: QTest::ignoreResultList);
311}
312
313void QTestLog::addPass(const char *msg)
314{
315 if (printAvailableTags)
316 return;
317
318 QTEST_ASSERT(msg);
319
320 ++QTest::passes;
321
322 FOREACH_TEST_LOGGER
323 logger->addIncident(type: QAbstractTestLogger::Pass, description: msg);
324}
325
326void QTestLog::addFail(const char *msg, const char *file, int line)
327{
328 QTEST_ASSERT(msg);
329
330 ++QTest::fails;
331
332 FOREACH_TEST_LOGGER
333 logger->addIncident(type: QAbstractTestLogger::Fail, description: msg, file, line);
334}
335
336void QTestLog::addXFail(const char *msg, const char *file, int line)
337{
338 QTEST_ASSERT(msg);
339 QTEST_ASSERT(file);
340
341 // Will be counted in addPass() if we get there.
342
343 FOREACH_TEST_LOGGER
344 logger->addIncident(type: QAbstractTestLogger::XFail, description: msg, file, line);
345}
346
347void QTestLog::addXPass(const char *msg, const char *file, int line)
348{
349 QTEST_ASSERT(msg);
350 QTEST_ASSERT(file);
351
352 ++QTest::fails;
353
354 FOREACH_TEST_LOGGER
355 logger->addIncident(type: QAbstractTestLogger::XPass, description: msg, file, line);
356}
357
358void QTestLog::addBPass(const char *msg)
359{
360 QTEST_ASSERT(msg);
361
362 ++QTest::blacklists;
363
364 FOREACH_TEST_LOGGER
365 logger->addIncident(type: QAbstractTestLogger::BlacklistedPass, description: msg);
366}
367
368void QTestLog::addBFail(const char *msg, const char *file, int line)
369{
370 QTEST_ASSERT(msg);
371 QTEST_ASSERT(file);
372
373 ++QTest::blacklists;
374
375 FOREACH_TEST_LOGGER
376 logger->addIncident(type: QAbstractTestLogger::BlacklistedFail, description: msg, file, line);
377}
378
379void QTestLog::addBXPass(const char *msg, const char *file, int line)
380{
381 QTEST_ASSERT(msg);
382 QTEST_ASSERT(file);
383
384 ++QTest::blacklists;
385
386 FOREACH_TEST_LOGGER
387 logger->addIncident(type: QAbstractTestLogger::BlacklistedXPass, description: msg, file, line);
388}
389
390void QTestLog::addBXFail(const char *msg, const char *file, int line)
391{
392 QTEST_ASSERT(msg);
393 QTEST_ASSERT(file);
394
395 // Will be counted in addBPass() if we get there.
396
397 FOREACH_TEST_LOGGER
398 logger->addIncident(type: QAbstractTestLogger::BlacklistedXFail, description: msg, file, line);
399}
400
401void QTestLog::addSkip(const char *msg, const char *file, int line)
402{
403 QTEST_ASSERT(msg);
404 QTEST_ASSERT(file);
405
406 ++QTest::skips;
407
408 FOREACH_TEST_LOGGER
409 logger->addMessage(type: QAbstractTestLogger::Skip, message: QString::fromUtf8(str: msg), file, line);
410}
411
412void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
413{
414 FOREACH_TEST_LOGGER
415 logger->addBenchmarkResult(result);
416}
417
418void QTestLog::startLogging()
419{
420 elapsedTotalTime.start();
421 elapsedFunctionTime.start();
422 FOREACH_TEST_LOGGER
423 logger->startLogging();
424 QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
425}
426
427void QTestLog::stopLogging()
428{
429 qInstallMessageHandler(QTest::oldMessageHandler);
430 FOREACH_TEST_LOGGER {
431 logger->stopLogging();
432 delete logger;
433 }
434 QTest::loggers()->clear();
435 saveCoverageTool(appname: QTestResult::currentAppName(), testfailed: failCount() != 0, installedTestCoverage: QTestLog::installedTestCoverage());
436}
437
438void QTestLog::addLogger(LogMode mode, const char *filename)
439{
440 if (filename && strcmp(s1: filename, s2: "-") == 0)
441 filename = nullptr;
442
443 QAbstractTestLogger *logger = nullptr;
444 switch (mode) {
445 case QTestLog::Plain:
446 logger = new QPlainTestLogger(filename);
447 break;
448 case QTestLog::CSV:
449 logger = new QCsvBenchmarkLogger(filename);
450 break;
451 case QTestLog::XML:
452 logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
453 break;
454 case QTestLog::LightXML:
455 logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
456 break;
457 case QTestLog::JUnitXML:
458 logger = new QJUnitTestLogger(filename);
459 break;
460 case QTestLog::TeamCity:
461 logger = new QTeamCityLogger(filename);
462 break;
463 case QTestLog::TAP:
464 logger = new QTapTestLogger(filename);
465 break;
466#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
467 case QTestLog::Apple:
468 logger = new QAppleTestLogger;
469 break;
470#endif
471#if defined(HAVE_XCTEST)
472 case QTestLog::XCTest:
473 logger = new QXcodeTestLogger;
474 break;
475#endif
476 }
477
478 QTEST_ASSERT(logger);
479 addLogger(logger);
480}
481
482/*!
483 \internal
484
485 Adds a new logger to the set of loggers that will be used
486 to report incidents and messages during testing.
487
488 The function takes ownership of the logger.
489*/
490void QTestLog::addLogger(QAbstractTestLogger *logger)
491{
492 QTEST_ASSERT(logger);
493 QTest::loggers()->append(t: logger);
494}
495
496int QTestLog::loggerCount()
497{
498 return QTest::loggers()->size();
499}
500
501bool QTestLog::loggerUsingStdout()
502{
503 FOREACH_TEST_LOGGER {
504 if (logger->isLoggingToStdout())
505 return true;
506 }
507
508 return false;
509}
510
511void QTestLog::warn(const char *msg, const char *file, int line)
512{
513 QTEST_ASSERT(msg);
514
515 FOREACH_TEST_LOGGER
516 logger->addMessage(type: QAbstractTestLogger::Warn, message: QString::fromUtf8(str: msg), file, line);
517}
518
519void QTestLog::info(const char *msg, const char *file, int line)
520{
521 QTEST_ASSERT(msg);
522
523 FOREACH_TEST_LOGGER
524 logger->addMessage(type: QAbstractTestLogger::Info, message: QString::fromUtf8(str: msg), file, line);
525}
526
527void QTestLog::setVerboseLevel(int level)
528{
529 QTest::verbosity = level;
530}
531
532int QTestLog::verboseLevel()
533{
534 return QTest::verbosity;
535}
536
537void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
538{
539 QTEST_ASSERT(msg);
540
541 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QString::fromUtf8(str: msg));
542}
543
544#if QT_CONFIG(regularexpression)
545void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
546{
547 QTEST_ASSERT(expression.isValid());
548
549 QTest::IgnoreResultList::append(list&: QTest::ignoreResultList, type, patternIn: QVariant(expression));
550}
551#endif // QT_CONFIG(regularexpression)
552
553void QTestLog::setMaxWarnings(int m)
554{
555 QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
556}
557
558bool QTestLog::printAvailableTags = false;
559
560void QTestLog::setPrintAvailableTagsMode()
561{
562 printAvailableTags = true;
563}
564
565int QTestLog::passCount()
566{
567 return QTest::passes;
568}
569
570int QTestLog::failCount()
571{
572 return QTest::fails;
573}
574
575int QTestLog::skipCount()
576{
577 return QTest::skips;
578}
579
580int QTestLog::blacklistCount()
581{
582 return QTest::blacklists;
583}
584
585int QTestLog::totalCount()
586{
587 return passCount() + failCount() + skipCount() + blacklistCount();
588}
589
590void QTestLog::resetCounters()
591{
592 QTest::passes = 0;
593 QTest::fails = 0;
594 QTest::skips = 0;
595}
596
597void QTestLog::setInstalledTestCoverage(bool installed)
598{
599 QTest::installedTestCoverage = installed;
600}
601
602bool QTestLog::installedTestCoverage()
603{
604 return QTest::installedTestCoverage;
605}
606
607qint64 QTestLog::nsecsTotalTime()
608{
609 return elapsedTotalTime.nsecsElapsed();
610}
611
612qint64 QTestLog::nsecsFunctionTime()
613{
614 return elapsedFunctionTime.nsecsElapsed();
615}
616
617QT_END_NAMESPACE
618
619#include "moc_qtestlog_p.cpp"
620

source code of qtbase/src/testlib/qtestlog.cpp