1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/private/qabstracttestlogger_p.h>
5#include <QtTest/qtestassert.h>
6#include <qbenchmark_p.h>
7#include <qtestresult_p.h>
8
9#include <QtCore/qbytearray.h>
10#include <QtCore/qstring.h>
11
12#include <cstdio>
13
14#include <stdio.h>
15#include <stdlib.h>
16#include <stdarg.h>
17
18#ifndef Q_OS_WIN
19#include <unistd.h>
20#endif
21
22#ifdef Q_OS_ANDROID
23#include <sys/stat.h>
24#endif
25
26QT_BEGIN_NAMESPACE
27/*!
28 \internal
29 \class QAbstractTestLogger
30 \inmodule QtTest
31 \brief Base class for test loggers
32
33 Implementations of logging for QtTest should implement all pure virtual
34 methods of this class and may implement the other virtual methods. This
35 class's documentation of each virtual method sets out how those
36 implementations are invoked by the QtTest code and offers guidance on how
37 the logging class should use the data. Actual implementations may have
38 different requirements - such as a file format with a defined schema, or a
39 target audience to serve - that affect how it interprets that guidance.
40*/
41
42/*!
43 \enum QAbstractTestLogger::IncidentTypes
44
45 \value Pass The test ran to completion successfully.
46 \value XFail The test failed a check that is known to fail; this failure
47 does not prevent successful completion of the test and may be
48 followed by further checks.
49 \value Fail The test fails.
50 \value XPass A check which was expected to fail actually passed. This is
51 counted as a failure, as it means whatever caused the known failure
52 no longer does, so the test needs an update.
53 \value Skip The current test ended prematurely, skipping some checks.
54 \value BlacklistedPass As Pass but the test was blacklisted.
55 \value BlacklistedXFail As XFail but the test was blacklisted.
56 \value BlacklistedFail As Fail but the test was blacklisted.
57 \value BlacklistedXPass As XPass but the test was blacklisted.
58
59 A test may also skip (see \l {QAbstractTestLogger::}{MessageTypes}). The
60 first of skip, Fail, XPass or the blacklisted equivalents of the last two to
61 arise is decisive for the outcome of the test: loggers which should only
62 report one outcome should thus record that as the outcome and either ignore
63 later incidents (or skips) in the same run of the test function or map them
64 to some form of message.
65
66 \note tests can be "blacklisted" when they are known to fail
67 unreliably. When testing is used to decide whether a change to the code
68 under test is acceptable, such failures are not automatic grounds for
69 rejecting the change, if the unreliable failure was known before the
70 change. QTest::qExec(), as a result, only returns a failing status code if
71 some non-blacklisted test failed. Logging backends may reasonably report a
72 blacklisted result just as they would report the non-blacklisted equivalent,
73 optionally with some annotation to indicate that the result should not be
74 taken as damning evidence against recent changes to the code under test.
75
76 \sa QAbstractTestLogger::addIncident()
77*/
78
79/*!
80 \enum QAbstractTestLogger::MessageTypes
81
82 The members whose names begin with \c Q describe messages that originate in
83 calls, by the test or code under test, to Qt logging functions (implemented
84 as macros) whose names are similar, with a \c q in place of the leading \c
85 Q. The other members describe messages generated internally by QtTest.
86
87 \value QInfo An informational message from qInfo().
88 \value QWarning A warning from qWarning().
89 \value QDebug A debug message from qDebug().
90 \value QCritical A critical error from qCritical().
91 \value QFatal A fatal error from qFatal(), or an unrecognised message from
92 the Qt logging functions.
93 \value Info Messages QtTest generates as requested by the \c{-v1} or \c{-v2}
94 command-line option being specified when running the test.
95 \value Warn A warning generated internally by QtTest
96
97 \note For these purposes, some utilities provided by QtTestlib as helper
98 functions to facilitate testing - such as \l QSignalSpy, \l
99 QTestAccessibility, \l QTest::qExtractTestData(), and the facilities to
100 deliver artificial mouse and keyboard events - are treated as test code,
101 rather than internal to QtTest; they call \l qWarning() and friends rather
102 than using the internal logging infrastructure, so that \l
103 QTest::ignoreMessage() can be used to anticipate the messages.
104
105 \sa QAbstractTestLogger::addMessage()
106*/
107
108/*!
109 Constructs the base-class parts of the logger.
110
111 Derived classes should pass this base-constructor the \a filename of the
112 file to which they shall log test results, or \nullptr to write to standard
113 output. The protected member \c stream is set to the open file descriptor.
114*/
115QAbstractTestLogger::QAbstractTestLogger(const char *filename)
116{
117 if (!filename) {
118 stream = stdout;
119 return;
120 }
121#if defined(_MSC_VER)
122 if (::fopen_s(&stream, filename, "wt")) {
123#else
124 stream = ::fopen(filename: filename, modes: "wt");
125 if (!stream) {
126#endif
127 fprintf(stderr, format: "Unable to open file for logging: %s\n", filename);
128 ::exit(status: 1);
129 }
130#ifdef Q_OS_ANDROID
131 else {
132 // Make sure output is world-readable on Android
133 ::chmod(filename, 0666);
134 }
135#endif
136}
137
138/*!
139 Destroys the logger object.
140
141 If the protected \c stream is not standard output, it is closed. In any
142 case it is cleared.
143*/
144QAbstractTestLogger::~QAbstractTestLogger()
145{
146 QTEST_ASSERT(stream);
147 if (stream != stdout)
148 fclose(stream: stream);
149 stream = nullptr;
150}
151
152/*!
153 Returns true if the logger supports repeated test runs.
154
155 Repetition of test runs is disabled by default, and can be enabled only for
156 test loggers that support it. Even if the logger may create syntactically
157 correct test reports, log-file analyzers may assume that test names are
158 unique within one report file.
159*/
160bool QAbstractTestLogger::isRepeatSupported() const
161{
162 return false;
163}
164
165/*!
166 Returns true if the \c output stream is standard output.
167*/
168bool QAbstractTestLogger::isLoggingToStdout() const
169{
170 return stream == stdout;
171}
172
173/*!
174 Helper utility to blot out unprintable characters in \a str.
175
176 Takes a \c{'\0'}-terminated mutable string and changes any characters of it
177 that are not suitable for printing to \c{'?'} characters.
178*/
179void QAbstractTestLogger::filterUnprintable(char *str) const
180{
181 unsigned char *idx = reinterpret_cast<unsigned char *>(str);
182 while (*idx) {
183 if (((*idx < 0x20 && *idx != '\n' && *idx != '\t') || *idx == 0x7f))
184 *idx = '?';
185 ++idx;
186 }
187}
188
189/*!
190 Convenience method to write \a msg to the output stream.
191
192 The output \a msg must be a \c{'\0'}-terminated string (and not \nullptr).
193 A copy of it is passed to \l filterUnprintable() and the result written to
194 the output \c stream, which is then flushed.
195*/
196void QAbstractTestLogger::outputString(const char *msg)
197{
198 QTEST_ASSERT(stream);
199 QTEST_ASSERT(msg);
200
201 char *filtered = new char[strlen(s: msg) + 1];
202 strcpy(dest: filtered, src: msg);
203 filterUnprintable(str: filtered);
204
205 ::fputs(s: filtered, stream: stream);
206 ::fflush(stream: stream);
207
208 delete [] filtered;
209}
210
211/*!
212 Called before the start of a test run.
213
214 This virtual method is called before the first tests are run. A logging
215 implementation might open a file, write some preamble, or prepare in other
216 ways, such as setting up initial values of variables. It can use the usual
217 Qt logging infrastucture, since it is also called before QtTest installs its
218 own custom message handler.
219
220 \sa stopLogging()
221*/
222void QAbstractTestLogger::startLogging()
223{
224}
225
226/*!
227 Called after the end of a test run.
228
229 This virtual method is called after all tests have run. A logging
230 implementation might collate information gathered from the run, write a
231 summary, or close a file. It can use the usual Qt logging infrastucture,
232 since it is also called after QtTest has restored the default message
233 handler it replaced with its own custom message handler.
234
235 \sa startLogging()
236*/
237void QAbstractTestLogger::stopLogging()
238{
239}
240
241void QAbstractTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &result)
242{
243 for (const auto &m : result)
244 addBenchmarkResult(result: m);
245}
246
247/*!
248 \fn void QAbstractTestLogger::enterTestFunction(const char *function)
249
250 This virtual method is called before each test function is invoked. It is
251 passed the name of the test function (without its class prefix) as \a
252 function. It is likewise called for \c{initTestCase()} at the start of
253 testing, after \l startLogging(), and for \c{cleanupTestCase()} at the end
254 of testing, in each case passing the name of the function. It is also called
255 with \nullptr as \a function after the last of these functions, or in the
256 event of an early end to testing, before \l stopLogging().
257
258 For data-driven test functions, this is called only once, before the data
259 function is called to set up the table of datasets and the test is run with
260 its first dataset.
261
262 Every logging implementation must implement this method. It shall typically
263 need to record the name of the function for later use in log messages.
264
265 \sa leaveTestFunction(), enterTestData()
266*/
267/*!
268 \fn void QAbstractTestLogger::leaveTestFunction()
269
270 This virtual method is called after a test function has completed, to match
271 \l enterTestFunction(). For data-driven test functions, this is called only
272 once, after the test is run with its last dataset.
273
274 Every logging implementation must implement this method. In some cases it
275 may be called more than once without an intervening call to \l
276 enterTestFunction(). In such cases, the implementation should ignore these
277 later calls, until the next call to enterTestFunction().
278
279 \sa enterTestFunction(), enterTestData()
280*/
281/*!
282 \fn void QAbstractTestLogger::enterTestData(QTestData *)
283
284 This virtual method is called before and after each call to a test
285 function. For a data-driven test, the call before is passed the name of the
286 test data row. This may combine a global data row name with a local data row
287 name. For non-data-driven tests and for the call after a test function,
288 \nullptr is passed
289
290 A logging implementation might chose to record the data row name for
291 reporting of results from the test for that data row. It should, in such a
292 case, clear its record of the name when called with \nullptr.
293
294 \sa enterTestFunction(), leaveTestFunction()
295*/
296/*!
297 \fn void QAbstractTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line)
298
299 This virtual method is called when an event occurs that relates to the
300 resolution of the test. The \a type indicates whether this was a pass, a
301 fail or a skip, whether a failure was expected, and whether the test being
302 run is blacklisted. The \a description may be empty (for a pass) or a
303 message describing the nature of the incident. Where the location in code of
304 the incident is known, it is indicated by \a file and \a line; otherwise,
305 these are \a nullptr and 0, respectively.
306
307 Every logging implementation must implement this method. Note that there are
308 circumstances where more than one incident may be reported, in this way, for
309 a single run of a test on a single dataset. It is the implementation's
310 responsibility to recognize such cases and decide what to do about them. For
311 purposes of counting resolutions of tests in the "Totals" report at the end
312 of a test run, QtTest considers the first incident (excluding XFail and its
313 blacklisted variant) decisive.
314
315 \sa addMessage(), addBenchmarkResult()
316*/
317/*!
318 \fn void QAbstractTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
319
320 This virtual method is called after a benchmark has been run enough times to
321 produce usable data. It is passed the median \a result from all cycles of
322 the code controlled by the test's QBENCHMARK loop.
323
324 Every logging implementation must implement this method.
325
326 \sa addIncident(), addMessage()
327*/
328/*!
329 \overload
330 \fn void QAbstractTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
331
332 This virtual method is called, via its \c QtMsgType overload, from the
333 custom message handler QtTest installs. It is also used to
334 warn about various situations detected by QtTest itself, such
335 as \e failure to see a message anticipated by QTest::ignoreMessage() and,
336 particularly when verbosity options have been enabled via the command-line,
337 to log some extra information.
338
339 Every logging implementation must implement this method. The \a type
340 indicates the category of message and the \a message is the content to be
341 reported. When the message is associated with specific code, the name of the
342 \a file and \a line number within it are also supplied; otherwise, these are
343 \nullptr and 0, respectively.
344
345 \sa QTest::ignoreMessage(), addIncident()
346*/
347
348/*!
349 \overload
350
351 This virtual method is called from the custom message handler QtTest
352 installs in place of Qt's default message handler for the duration of
353 testing, unless QTest::ignoreMessage() was used to ignore it, or too many
354 messages have previously been processed. (The limiting number of messages is
355 controlled by the -maxwarnings option to a test and defaults to 2002.)
356
357 Logging implementations should not normally need to override this method.
358 The base implementation converts \a type to the matching \l MessageType,
359 formats the given \a message suitably for the specified \a context, and
360 forwards the converted type and formatted message to the overload that takes
361 MessageType and QString.
362
363 \sa QTest::ignoreMessage(), addIncident()
364*/
365void QAbstractTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context,
366 const QString &message)
367{
368 QAbstractTestLogger::MessageTypes messageType = [=]() {
369 switch (type) {
370 case QtDebugMsg: return QAbstractTestLogger::QDebug;
371 case QtInfoMsg: return QAbstractTestLogger::QInfo;
372 case QtCriticalMsg: return QAbstractTestLogger::QCritical;
373 case QtWarningMsg: return QAbstractTestLogger::QWarning;
374 case QtFatalMsg: return QAbstractTestLogger::QFatal;
375 }
376 Q_UNREACHABLE_RETURN(QAbstractTestLogger::QFatal);
377 }();
378
379 QString formattedMessage = qFormatLogMessage(type, context, buf: message);
380
381 // Note that we explicitly ignore the file and line of the context here,
382 // as that's what QTest::messageHandler used to do when calling the same
383 // overload directly.
384 addMessage(type: messageType, message: formattedMessage);
385}
386
387namespace
388{
389 constexpr int MAXSIZE = 1024 * 1024 * 2;
390}
391
392namespace QTest
393{
394
395/*!
396 \fn int QTest::qt_asprintf(QTestCharBuffer *buf, const char *format, ...);
397 \internal
398 */
399int qt_asprintf(QTestCharBuffer *str, const char *format, ...)
400{
401 Q_ASSERT(str);
402 int size = str->size();
403 Q_ASSERT(size > 0);
404
405 va_list ap;
406 int res = 0;
407
408 do {
409 va_start(ap, format);
410 res = std::vsnprintf(s: str->data(), maxlen: size, format: format, arg: ap);
411 va_end(ap);
412 // vsnprintf() reliably '\0'-terminates
413 Q_ASSERT(res < 0 || str->data()[res < size ? res : size - 1] == '\0');
414 // Note, we're assuming that a result of -1 is always due to running out of space.
415 if (res >= 0 && res < size) // Success
416 break;
417
418 // Buffer wasn't big enough, try again:
419 size *= 2;
420 // If too large or out of memory, take what we have:
421 } while (size <= MAXSIZE && str->reset(newSize: size));
422
423 return res;
424}
425
426}
427
428namespace QTestPrivate
429{
430
431void generateTestIdentifier(QTestCharBuffer *identifier, int parts)
432{
433 const char *testObject = parts & TestObject ? QTestResult::currentTestObjectName() : "";
434 const char *testFunction = parts & TestFunction ? (QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc") : "";
435 const char *objectFunctionFiller = parts & TestObject && parts & (TestFunction | TestDataTag) ? "::" : "";
436 const char *testFuctionStart = parts & TestFunction ? "(" : "";
437 const char *testFuctionEnd = parts & TestFunction ? ")" : "";
438
439 const char *dataTag = (parts & TestDataTag) && QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
440 const char *globalDataTag = (parts & TestDataTag) && QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
441 const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : "";
442
443 QTest::qt_asprintf(str: identifier, format: "%s%s%s%s%s%s%s%s",
444 testObject, objectFunctionFiller, testFunction, testFuctionStart,
445 globalDataTag, tagFiller, dataTag, testFuctionEnd);
446}
447
448// strcat() for QTestCharBuffer objects:
449bool appendCharBuffer(QTestCharBuffer *accumulator, const QTestCharBuffer &more)
450{
451 const auto bufsize = [](const QTestCharBuffer &buf) -> int {
452 const int max = buf.size();
453 return max > 0 ? int(qstrnlen(str: buf.constData(), maxlen: max)) : 0;
454 };
455 const int extra = bufsize(more);
456 if (extra <= 0)
457 return true; // Nothing to do, fatuous success
458
459 const int oldsize = bufsize(*accumulator);
460 const int newsize = oldsize + extra + 1; // 1 for final '\0'
461 if (newsize > MAXSIZE || !accumulator->resize(newSize: newsize))
462 return false; // too big or unable to grow
463
464 char *tail = accumulator->data() + oldsize;
465 memcpy(dest: tail, src: more.constData(), n: extra);
466 tail[extra] = '\0';
467 return true;
468}
469
470}
471
472QT_END_NAMESPACE
473

source code of qtbase/src/testlib/qabstracttestlogger.cpp