1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/private/qabstracttestlogger_p.h>
5#include <QtTest/qtestassert.h>
6#include <qbenchmark_p.h>
7#include <qtestresult_p.h>
8
9#include <QtCore/qbytearray.h>
10#include <QtCore/qstring.h>
11
12#include <stdio.h>
13#include <stdlib.h>
14#include <stdarg.h>
15
16#ifndef Q_OS_WIN
17#include <unistd.h>
18#endif
19
20#ifdef Q_OS_ANDROID
21#include <sys/stat.h>
22#endif
23
24QT_BEGIN_NAMESPACE
25/*!
26 \internal
27 \class QAbstractTestLogger
28 \inmodule QtTest
29 \brief Base class for test loggers
30
31 Implementations of logging for QtTest should implement all pure virtual
32 methods of this class and may implement the other virtual methods. This
33 class's documentation of each virtual method sets out how those
34 implementations are invoked by the QtTest code and offers guidance on how
35 the logging class should use the data. Actual implementations may have
36 different requirements - such as a file format with a defined schema, or a
37 target audience to serve - that affect how it interprets that guidance.
38*/
39
40/*!
41 \enum QAbstractTestLogger::IncidentTypes
42
43 \value Pass The test ran to completion successfully.
44 \value XFail The test failed a check that is known to fail; this failure
45 does not prevent successful completion of the test and may be
46 followed by further checks.
47 \value Fail The test fails.
48 \value XPass A check which was expected to fail actually passed. This is
49 counted as a failure, as it means whatever caused the known failure
50 no longer does, so the test needs an update.
51 \value Skip The current test ended prematurely, skipping some checks.
52 \value BlacklistedPass As Pass but the test was blacklisted.
53 \value BlacklistedXFail As XFail but the test was blacklisted.
54 \value BlacklistedFail As Fail but the test was blacklisted.
55 \value BlacklistedXPass As XPass but the test was blacklisted.
56
57 A test may also skip (see \l {QAbstractTestLogger::}{MessageTypes}). The
58 first of skip, Fail, XPass or the blacklisted equivalents of the last two to
59 arise is decisive for the outcome of the test: loggers which should only
60 report one outcome should thus record that as the outcome and either ignore
61 later incidents (or skips) in the same run of the test function or map them
62 to some form of message.
63
64 \note tests can be "blacklisted" when they are known to fail
65 unreliably. When testing is used to decide whether a change to the code
66 under test is acceptable, such failures are not automatic grounds for
67 rejecting the change, if the unreliable failure was known before the
68 change. QTest::qExec(), as a result, only returns a failing status code if
69 some non-blacklisted test failed. Logging backends may reasonably report a
70 blacklisted result just as they would report the non-blacklisted equivalent,
71 optionally with some annotation to indicate that the result should not be
72 taken as damning evidence against recent changes to the code under test.
73
74 \sa QAbstractTestLogger::addIncident()
75*/
76
77/*!
78 \enum QAbstractTestLogger::MessageTypes
79
80 The members whose names begin with \c Q describe messages that originate in
81 calls, by the test or code under test, to Qt logging functions (implemented
82 as macros) whose names are similar, with a \c q in place of the leading \c
83 Q. The other members describe messages generated internally by QtTest.
84
85 \value QInfo An informational message from qInfo().
86 \value QWarning A warning from qWarning().
87 \value QDebug A debug message from qDebug().
88 \value QCritical A critical error from qCritical().
89 \value QFatal A fatal error from qFatal(), or an unrecognised message from
90 the Qt logging functions.
91 \value Info Messages QtTest generates as requested by the \c{-v1} or \c{-v2}
92 command-line option being specified when running the test.
93 \value Warn A warning generated internally by QtTest
94
95 \note For these purposes, some utilities provided by QtTestlib as helper
96 functions to facilitate testing - such as \l QSignalSpy, \l
97 QTestAccessibility, \l QTest::qExtractTestData(), and the facilities to
98 deliver artificial mouse and keyboard events - are treated as test code,
99 rather than internal to QtTest; they call \l qWarning() and friends rather
100 than using the internal logging infrastructure, so that \l
101 QTest::ignoreMessage() can be used to anticipate the messages.
102
103 \sa QAbstractTestLogger::addMessage()
104*/
105
106/*!
107 Constructs the base-class parts of the logger.
108
109 Derived classes should pass this base-constructor the \a filename of the
110 file to which they shall log test results, or \nullptr to write to standard
111 output. The protected member \c stream is set to the open file descriptor.
112*/
113QAbstractTestLogger::QAbstractTestLogger(const char *filename)
114{
115 if (!filename) {
116 stream = stdout;
117 return;
118 }
119#if defined(_MSC_VER)
120 if (::fopen_s(&stream, filename, "wt")) {
121#else
122 stream = ::fopen(filename: filename, modes: "wt");
123 if (!stream) {
124#endif
125 fprintf(stderr, format: "Unable to open file for logging: %s\n", filename);
126 ::exit(status: 1);
127 }
128#ifdef Q_OS_ANDROID
129 else {
130 // Make sure output is world-readable on Android
131 ::chmod(filename, 0666);
132 }
133#endif
134}
135
136/*!
137 Destroys the logger object.
138
139 If the protected \c stream is not standard output, it is closed. In any
140 case it is cleared.
141*/
142QAbstractTestLogger::~QAbstractTestLogger()
143{
144 QTEST_ASSERT(stream);
145 if (stream != stdout)
146 fclose(stream: stream);
147 stream = nullptr;
148}
149
150/*!
151 Returns true if the \c output stream is standard output.
152*/
153bool QAbstractTestLogger::isLoggingToStdout() const
154{
155 return stream == stdout;
156}
157
158/*!
159 Helper utility to blot out unprintable characters in \a str.
160
161 Takes a \c{'\0'}-terminated mutable string and changes any characters of it
162 that are not suitable for printing to \c{'?'} characters.
163*/
164void QAbstractTestLogger::filterUnprintable(char *str) const
165{
166 unsigned char *idx = reinterpret_cast<unsigned char *>(str);
167 while (*idx) {
168 if (((*idx < 0x20 && *idx != '\n' && *idx != '\t') || *idx == 0x7f))
169 *idx = '?';
170 ++idx;
171 }
172}
173
174/*!
175 Convenience method to write \a msg to the output stream.
176
177 The output \a msg must be a \c{'\0'}-terminated string (and not \nullptr).
178 A copy of it is passed to \l filterUnprintable() and the result written to
179 the output \c stream, which is then flushed.
180*/
181void QAbstractTestLogger::outputString(const char *msg)
182{
183 QTEST_ASSERT(stream);
184 QTEST_ASSERT(msg);
185
186 char *filtered = new char[strlen(s: msg) + 1];
187 strcpy(dest: filtered, src: msg);
188 filterUnprintable(str: filtered);
189
190 ::fputs(s: filtered, stream: stream);
191 ::fflush(stream: stream);
192
193 delete [] filtered;
194}
195
196/*!
197 Called before the start of a test run.
198
199 This virtual method is called before the first tests are run. A logging
200 implementation might open a file, write some preamble, or prepare in other
201 ways, such as setting up initial values of variables. It can use the usual
202 Qt logging infrastucture, since it is also called before QtTest installs its
203 own custom message handler.
204
205 \sa stopLogging()
206*/
207void QAbstractTestLogger::startLogging()
208{
209}
210
211/*!
212 Called after the end of a test run.
213
214 This virtual method is called after all tests have run. A logging
215 implementation might collate information gathered from the run, write a
216 summary, or close a file. It can use the usual Qt logging infrastucture,
217 since it is also called after QtTest has restored the default message
218 handler it replaced with its own custom message handler.
219
220 \sa startLogging()
221*/
222void QAbstractTestLogger::stopLogging()
223{
224}
225
226void QAbstractTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &result)
227{
228 for (const auto &m : result)
229 addBenchmarkResult(result: m);
230}
231
232/*!
233 \fn void QAbstractTestLogger::enterTestFunction(const char *function)
234
235 This virtual method is called before each test function is invoked. It is
236 passed the name of the test function (without its class prefix) as \a
237 function. It is likewise called for \c{initTestCase()} at the start of
238 testing, after \l startLogging(), and for \c{cleanupTestCase()} at the end
239 of testing, in each case passing the name of the function. It is also called
240 with \nullptr as \a function after the last of these functions, or in the
241 event of an early end to testing, before \l stopLogging().
242
243 For data-driven test functions, this is called only once, before the data
244 function is called to set up the table of datasets and the test is run with
245 its first dataset.
246
247 Every logging implementation must implement this method. It shall typically
248 need to record the name of the function for later use in log messages.
249
250 \sa leaveTestFunction(), enterTestData()
251*/
252/*!
253 \fn void QAbstractTestLogger::leaveTestFunction()
254
255 This virtual method is called after a test function has completed, to match
256 \l enterTestFunction(). For data-driven test functions, this is called only
257 once, after the test is run with its last dataset.
258
259 Every logging implementation must implement this method. In some cases it
260 may be called more than once without an intervening call to \l
261 enterTestFunction(). In such cases, the implementation should ignore these
262 later calls, until the next call to enterTestFunction().
263
264 \sa enterTestFunction(), enterTestData()
265*/
266/*!
267 \fn void QAbstractTestLogger::enterTestData(QTestData *)
268
269 This virtual method is called before and after each call to a test
270 function. For a data-driven test, the call before is passed the name of the
271 test data row. This may combine a global data row name with a local data row
272 name. For non-data-driven tests and for the call after a test function,
273 \nullptr is passed
274
275 A logging implementation might chose to record the data row name for
276 reporting of results from the test for that data row. It should, in such a
277 case, clear its record of the name when called with \nullptr.
278
279 \sa enterTestFunction(), leaveTestFunction()
280*/
281/*!
282 \fn void QAbstractTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line)
283
284 This virtual method is called when an event occurs that relates to the
285 resolution of the test. The \a type indicates whether this was a pass, a
286 fail or a skip, whether a failure was expected, and whether the test being
287 run is blacklisted. The \a description may be empty (for a pass) or a
288 message describing the nature of the incident. Where the location in code of
289 the incident is known, it is indicated by \a file and \a line; otherwise,
290 these are \a nullptr and 0, respectively.
291
292 Every logging implementation must implement this method. Note that there are
293 circumstances where more than one incident may be reported, in this way, for
294 a single run of a test on a single dataset. It is the implementation's
295 responsibility to recognize such cases and decide what to do about them. For
296 purposes of counting resolutions of tests in the "Totals" report at the end
297 of a test run, QtTest considers the first incident (excluding XFail and its
298 blacklisted variant) decisive.
299
300 \sa addMessage(), addBenchmarkResult()
301*/
302/*!
303 \fn void QAbstractTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
304
305 This virtual method is called after a benchmark has been run enough times to
306 produce usable data. It is passed the median \a result from all cycles of
307 the code controlled by the test's QBENCHMARK loop.
308
309 Every logging implementation must implement this method.
310
311 \sa addIncident(), addMessage()
312*/
313/*!
314 \overload
315 \fn void QAbstractTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
316
317 This virtual method is called, via its \c QtMsgType overload, from the
318 custom message handler QtTest installs. It is also used to
319 warn about various situations detected by QtTest itself, such
320 as \e failure to see a message anticipated by QTest::ignoreMessage() and,
321 particularly when verbosity options have been enabled via the command-line,
322 to log some extra information.
323
324 Every logging implementation must implement this method. The \a type
325 indicates the category of message and the \a message is the content to be
326 reported. When the message is associated with specific code, the name of the
327 \a file and \a line number within it are also supplied; otherwise, these are
328 \nullptr and 0, respectively.
329
330 \sa QTest::ignoreMessage(), addIncident()
331*/
332
333/*!
334 \overload
335
336 This virtual method is called from the custom message handler QtTest
337 installs in place of Qt's default message handler for the duration of
338 testing, unless QTest::ignoreMessage() was used to ignore it, or too many
339 messages have previously been processed. (The limiting number of messages is
340 controlled by the -maxwarnings option to a test and defaults to 2002.)
341
342 Logging implementations should not normally need to override this method.
343 The base implementation converts \a type to the matching \l MessageType,
344 formats the given \a message suitably for the specified \a context, and
345 forwards the converted type and formatted message to the overload that takes
346 MessageType and QString.
347
348 \sa QTest::ignoreMessage(), addIncident()
349*/
350void QAbstractTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context,
351 const QString &message)
352{
353 QAbstractTestLogger::MessageTypes messageType = [=]() {
354 switch (type) {
355 case QtDebugMsg: return QAbstractTestLogger::QDebug;
356 case QtInfoMsg: return QAbstractTestLogger::QInfo;
357 case QtCriticalMsg: return QAbstractTestLogger::QCritical;
358 case QtWarningMsg: return QAbstractTestLogger::QWarning;
359 case QtFatalMsg: return QAbstractTestLogger::QFatal;
360 }
361 Q_UNREACHABLE_RETURN(QAbstractTestLogger::QFatal);
362 }();
363
364 QString formattedMessage = qFormatLogMessage(type, context, buf: message);
365
366 // Note that we explicitly ignore the file and line of the context here,
367 // as that's what QTest::messageHandler used to do when calling the same
368 // overload directly.
369 addMessage(type: messageType, message: formattedMessage);
370}
371
372namespace
373{
374 constexpr int MAXSIZE = 1024 * 1024 * 2;
375}
376
377namespace QTest
378{
379
380/*!
381 \fn int QTest::qt_asprintf(QTestCharBuffer *buf, const char *format, ...);
382 \internal
383 */
384int qt_asprintf(QTestCharBuffer *str, const char *format, ...)
385{
386 Q_ASSERT(str);
387 int size = str->size();
388 Q_ASSERT(size > 0);
389
390 va_list ap;
391 int res = 0;
392
393 do {
394 va_start(ap, format);
395 res = qvsnprintf(str: str->data(), n: size, fmt: format, ap);
396 va_end(ap);
397 // vsnprintf() reliably '\0'-terminates
398 Q_ASSERT(res < 0 || str->data()[res < size ? res : size - 1] == '\0');
399 // Note, we're assuming that a result of -1 is always due to running out of space.
400 if (res >= 0 && res < size) // Success
401 break;
402
403 // Buffer wasn't big enough, try again:
404 size *= 2;
405 // If too large or out of memory, take what we have:
406 } while (size <= MAXSIZE && str->reset(newSize: size));
407
408 return res;
409}
410
411}
412
413namespace QTestPrivate
414{
415
416void generateTestIdentifier(QTestCharBuffer *identifier, int parts)
417{
418 const char *testObject = parts & TestObject ? QTestResult::currentTestObjectName() : "";
419 const char *testFunction = parts & TestFunction ? (QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc") : "";
420 const char *objectFunctionFiller = parts & TestObject && parts & (TestFunction | TestDataTag) ? "::" : "";
421 const char *testFuctionStart = parts & TestFunction ? "(" : "";
422 const char *testFuctionEnd = parts & TestFunction ? ")" : "";
423
424 const char *dataTag = (parts & TestDataTag) && QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
425 const char *globalDataTag = (parts & TestDataTag) && QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
426 const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : "";
427
428 QTest::qt_asprintf(str: identifier, format: "%s%s%s%s%s%s%s%s",
429 testObject, objectFunctionFiller, testFunction, testFuctionStart,
430 globalDataTag, tagFiller, dataTag, testFuctionEnd);
431}
432
433// strcat() for QTestCharBuffer objects:
434bool appendCharBuffer(QTestCharBuffer *accumulator, const QTestCharBuffer &more)
435{
436 const auto bufsize = [](const QTestCharBuffer &buf) -> int {
437 const int max = buf.size();
438 return max > 0 ? int(qstrnlen(str: buf.constData(), maxlen: max)) : 0;
439 };
440 const int extra = bufsize(more);
441 if (extra <= 0)
442 return true; // Nothing to do, fatuous success
443
444 const int oldsize = bufsize(*accumulator);
445 const int newsize = oldsize + extra + 1; // 1 for final '\0'
446 if (newsize > MAXSIZE || !accumulator->resize(newSize: newsize))
447 return false; // too big or unable to grow
448
449 char *tail = accumulator->data() + oldsize;
450 memcpy(dest: tail, src: more.constData(), n: extra);
451 tail[extra] = '\0';
452 return true;
453}
454
455}
456
457QT_END_NAMESPACE
458

source code of qtbase/src/testlib/qabstracttestlogger.cpp