1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/private/qtestresult_p.h>
5#include <QtTest/qtestassert.h>
6#include <QtTest/private/qtestlog_p.h>
7#include <QtTest/private/qplaintestlogger_p.h>
8#include <QtTest/private/qbenchmark_p.h>
9#include <QtTest/private/qbenchmarkmetric_p.h>
10
11#include <QtCore/private/qlogging_p.h>
12
13#include <array>
14#include <cstdio>
15
16#include <stdio.h>
17#include <stdlib.h>
18#include <string.h>
19
20#include <QtCore/QByteArray>
21#include <QtCore/qmath.h>
22#include <QtCore/QLibraryInfo>
23
24#ifdef Q_OS_ANDROID
25# include <android/log.h>
26#endif
27
28#ifdef Q_OS_WIN
29# include <qt_windows.h>
30#endif
31
32QT_BEGIN_NAMESPACE
33
34using namespace Qt::StringLiterals;
35
36namespace {
37static const char multiplePrefixes[] = "\0kMGTPE"; // kilo, mega, giga, tera, peta, exa
38static const char submultiplePrefixes[] = "afpnum"; // atto, femto, pico, nano, micro, milli
39
40template <int N> struct FixedBufString
41{
42 static constexpr size_t MaxSize = N;
43 size_t used = 0;
44 std::array<char, N + 2> buf; // for the newline and terminating null
45 FixedBufString()
46 {
47 clear();
48 }
49 void clear()
50 {
51 used = 0;
52 buf[0] = '\0';
53 }
54
55 operator const char *() const
56 {
57 return buf.data();
58 }
59
60 void append(const char *text)
61 {
62 size_t len = qMin(a: strlen(s: text), b: MaxSize - used);
63 memcpy(buf.data() + used, text, len);
64 used += len;
65 buf[used] = '\0';
66 }
67
68 template <typename... Args> void appendf(const char *format, Args... args)
69 {
70 // vsnprintf includes the terminating null
71 used += std::snprintf(s: buf.data() + used, maxlen: MaxSize - used + 1, format: format,
72 args...);
73 }
74
75 template <int Power = 1000> void appendScaled(qreal value, const char *unit)
76 {
77 char prefix[2] = {};
78 qreal v = qAbs(t: value);
79 qint64 ratio;
80 if (v < 1 && Power == 1000) {
81 const char *prefixes = submultiplePrefixes;
82 ratio = qreal(std::atto::num) / qreal(std::atto::den);
83 while (value * ratio > 1000 && *prefixes) {
84 ++prefixes;
85 ratio *= 1000;
86 }
87 prefix[0] = *prefixes;
88 } else {
89 const char *prefixes = multiplePrefixes;
90 ratio = 1;
91 while (value > 1000 * ratio) { // yes, even for binary
92 ++prefixes;
93 ratio *= Power;
94 }
95 prefix[0] = *prefixes;
96 }
97
98 // adjust the value by the ratio
99 value /= ratio;
100 appendf(", %.3g %s%s", value, prefix, unit);
101 }
102};
103} // unnamed namespace
104
105namespace QTest {
106
107 static const char *ptIncidentType2String(QAbstractTestLogger::IncidentTypes type)
108 {
109 switch (type) {
110 case QAbstractTestLogger::Skip:
111 return "SKIP ";
112 case QAbstractTestLogger::Pass:
113 return "PASS ";
114 case QAbstractTestLogger::XFail:
115 return "XFAIL ";
116 case QAbstractTestLogger::Fail:
117 return "FAIL! ";
118 case QAbstractTestLogger::XPass:
119 return "XPASS ";
120 case QAbstractTestLogger::BlacklistedPass:
121 return "BPASS ";
122 case QAbstractTestLogger::BlacklistedFail:
123 return "BFAIL ";
124 case QAbstractTestLogger::BlacklistedXPass:
125 return "BXPASS ";
126 case QAbstractTestLogger::BlacklistedXFail:
127 return "BXFAIL ";
128 }
129 Q_UNREACHABLE_RETURN(nullptr);
130 }
131
132 static const char *benchmarkResult2String()
133 {
134 return "RESULT ";
135 }
136
137 static const char *ptMessageType2String(QAbstractTestLogger::MessageTypes type)
138 {
139 switch (type) {
140 case QAbstractTestLogger::QDebug:
141 return "QDEBUG ";
142 case QAbstractTestLogger::QInfo:
143 return "QINFO ";
144 case QAbstractTestLogger::QWarning:
145 return "QWARN ";
146 case QAbstractTestLogger::QCritical:
147 return "QCRITICAL";
148 case QAbstractTestLogger::QFatal:
149 return "QFATAL ";
150 case QAbstractTestLogger::Info:
151 return "INFO ";
152 case QAbstractTestLogger::Warn:
153 return "WARNING";
154 }
155 Q_UNREACHABLE_RETURN(nullptr);
156 }
157
158 template <typename T>
159 static int countSignificantDigits(T num)
160 {
161 if (num <= 0)
162 return 0;
163
164 int digits = 0;
165 qreal divisor = 1;
166
167 while (num / divisor >= 1) {
168 divisor *= 10;
169 ++digits;
170 }
171
172 return digits;
173 }
174
175 // Pretty-prints a benchmark result using the given number of digits.
176 template <typename T> QByteArray formatResult(T number, int significantDigits)
177 {
178 if (number < T(0))
179 return "NAN";
180 if (number == T(0))
181 return "0";
182
183 QByteArray beforeDecimalPoint = QByteArray::number(qint64(number), format: 'f', precision: 0);
184 QByteArray afterDecimalPoint = QByteArray::number(number, 'f', 20);
185 afterDecimalPoint.remove(index: 0, len: beforeDecimalPoint.size() + 1);
186
187 int beforeUse = qMin(a: beforeDecimalPoint.size(), b: significantDigits);
188 int beforeRemove = beforeDecimalPoint.size() - beforeUse;
189
190 // Replace insignificant digits before the decimal point with zeros.
191 beforeDecimalPoint.chop(n: beforeRemove);
192 for (int i = 0; i < beforeRemove; ++i) {
193 beforeDecimalPoint.append(c: u'0');
194 }
195
196 int afterUse = significantDigits - beforeUse;
197
198 // leading zeroes after the decimal point does not count towards the digit use.
199 if (beforeDecimalPoint == "0" && !afterDecimalPoint.isEmpty()) {
200 ++afterUse;
201
202 int i = 0;
203 while (i < afterDecimalPoint.size() && afterDecimalPoint.at(i) == '0')
204 ++i;
205
206 afterUse += i;
207 }
208
209 int afterRemove = afterDecimalPoint.size() - afterUse;
210 afterDecimalPoint.chop(n: afterRemove);
211
212 char separator = ',';
213 char decimalPoint = '.';
214
215 // insert thousands separators
216 int length = beforeDecimalPoint.size();
217 for (int i = beforeDecimalPoint.size() -1; i >= 1; --i) {
218 if ((length - i) % 3 == 0)
219 beforeDecimalPoint.insert(i, c: separator);
220 }
221
222 QByteArray print;
223 print = beforeDecimalPoint;
224 if (afterUse > 0)
225 print.append(c: decimalPoint);
226
227 print += afterDecimalPoint;
228
229
230 return print;
231 }
232}
233
234/*! \internal
235 \class QPlainTestLogger
236 \inmodule QtTest
237
238 QPlainTestLogger implements basic logging of test results.
239
240 The format is Qt-specific and aims to be easy to read.
241*/
242
243void QPlainTestLogger::outputMessage(const char *str)
244{
245#if defined(Q_OS_WIN)
246 // Log to system log only if output is not redirected and stderr not preferred
247 if (stream == stdout && !QtPrivate::shouldLogToStderr()) {
248 OutputDebugStringA(str);
249 return;
250 }
251#elif defined(Q_OS_ANDROID)
252 __android_log_write(ANDROID_LOG_INFO, "QTestLib", str);
253#endif
254 outputString(msg: str);
255}
256
257void QPlainTestLogger::printMessage(MessageSource source, const char *type, const char *msg,
258 const char *file, int line)
259{
260 QTEST_ASSERT(type);
261 QTEST_ASSERT(msg);
262
263 QTestCharBuffer messagePrefix;
264
265 QTestCharBuffer messageLocation;
266#ifdef Q_OS_WIN
267 constexpr const char *INCIDENT_LOCATION_STR = "\n%s(%d) : failure location";
268 constexpr const char *OTHER_LOCATION_STR = "\n%s(%d) : message location";
269#else
270 constexpr const char *INCIDENT_LOCATION_STR = "\n Loc: [%s(%d)]";
271 constexpr const char *OTHER_LOCATION_STR = INCIDENT_LOCATION_STR;
272#endif
273
274 if (file) {
275 switch (source) {
276 case MessageSource::Incident:
277 QTest::qt_asprintf(buf: &messageLocation, format: INCIDENT_LOCATION_STR, file, line);
278 break;
279 case MessageSource::Other:
280 QTest::qt_asprintf(buf: &messageLocation, format: OTHER_LOCATION_STR, file, line);
281 break;
282 }
283 }
284
285 const char *msgFiller = msg[0] ? " " : "";
286 QTestCharBuffer testIdentifier;
287 QTestPrivate::generateTestIdentifier(identifier: &testIdentifier);
288 QTest::qt_asprintf(buf: &messagePrefix, format: "%s: %s%s%s%s\n",
289 type, testIdentifier.data(), msgFiller, msg, messageLocation.data());
290
291 // In colored mode, printf above stripped our nonprintable control characters.
292 // Put them back.
293 memcpy(dest: messagePrefix.data(), src: type, n: strlen(s: type));
294
295 outputMessage(str: messagePrefix.data());
296}
297
298void QPlainTestLogger::printBenchmarkResultsHeader(const QBenchmarkResult &result)
299{
300 FixedBufString<1022> buf;
301 buf.appendf(format: "%s: %s::%s", args: QTest::benchmarkResult2String(),
302 args: QTestResult::currentTestObjectName(), args: result.context.slotName.toLatin1().data());
303
304 QByteArray tag = QTestResult::currentDataTag();
305 QByteArray gtag = QTestResult::currentGlobalDataTag();
306
307 if (!gtag.isEmpty() && !tag.isEmpty())
308 buf.appendf(format: ":\"%s:%s\":\n", args: gtag.constData(), args: tag.constData());
309 else if (!gtag.isEmpty())
310 buf.appendf(format: ":\"%s\":\n", args: gtag.constData());
311 else if (!tag.isEmpty())
312 buf.appendf(format: ":\"%s\":\n", args: tag.constData());
313 else
314 buf.append(text: ":\n");
315 outputMessage(str: buf);
316}
317
318void QPlainTestLogger::printBenchmarkResults(const QList<QBenchmarkResult> &results)
319{
320 using namespace std::chrono;
321 FixedBufString<1022> buf;
322 auto findResultFor = [&results](QTest::QBenchmarkMetric metric) -> std::optional<qreal> {
323 for (const QBenchmarkResult &result : results) {
324 if (result.measurement.metric == metric)
325 return result.measurement.value;
326 }
327 return std::nullopt;
328 };
329
330 // we need the execution time quite often, so find it first
331 qreal executionTime = 0;
332 if (auto ns = findResultFor(QTest::WalltimeNanoseconds))
333 executionTime = *ns / (1000 * 1000 * 1000);
334 else if (auto ms = findResultFor(QTest::WalltimeMilliseconds))
335 executionTime = *ms / 1000;
336
337 for (const QBenchmarkResult &result : results) {
338 buf.clear();
339
340 const char * unitText = QTest::benchmarkMetricUnit(metric: result.measurement.metric);
341 int significantDigits = QTest::countSignificantDigits(num: result.measurement.value);
342 qreal valuePerIteration = qreal(result.measurement.value) / qreal(result.iterations);
343 buf.appendf(format: " %s %s%s", args: QTest::formatResult(number: valuePerIteration, significantDigits).constData(),
344 args: unitText, args: result.setByMacro ? " per iteration" : "");
345
346 switch (result.measurement.metric) {
347 case QTest::BitsPerSecond:
348 // for bits/s, we'll use powers of 10 (1 Mbit/s = 1000 kbit/s = 1000000 bit/s)
349 buf.appendScaled<1000>(value: result.measurement.value, unit: "bit/s");
350 break;
351 case QTest::BytesPerSecond:
352 // for B/s, we'll use powers of 2 (1 MB/s = 1024 kB/s = 1048576 B/s)
353 buf.appendScaled<1024>(value: result.measurement.value, unit: "B/s");
354 break;
355
356 case QTest::CPUCycles:
357 case QTest::RefCPUCycles:
358 if (!qIsNull(d: executionTime))
359 buf.appendScaled(value: result.measurement.value / executionTime, unit: "Hz");
360 break;
361
362 case QTest::Instructions:
363 if (auto cycles = findResultFor(QTest::CPUCycles)) {
364 buf.appendf(format: ", %.3f instr/cycle", args: result.measurement.value / *cycles);
365 break;
366 }
367 Q_FALLTHROUGH();
368
369 case QTest::InstructionReads:
370 case QTest::Events:
371 case QTest::BytesAllocated:
372 case QTest::CPUMigrations:
373 case QTest::BusCycles:
374 case QTest::StalledCycles:
375 case QTest::BranchInstructions:
376 case QTest::BranchMisses:
377 case QTest::CacheReferences:
378 case QTest::CacheReads:
379 case QTest::CacheWrites:
380 case QTest::CachePrefetches:
381 case QTest::CacheMisses:
382 case QTest::CacheReadMisses:
383 case QTest::CacheWriteMisses:
384 case QTest::CachePrefetchMisses:
385 case QTest::ContextSwitches:
386 case QTest::PageFaults:
387 case QTest::MinorPageFaults:
388 case QTest::MajorPageFaults:
389 case QTest::AlignmentFaults:
390 case QTest::EmulationFaults:
391 if (!qIsNull(d: executionTime))
392 buf.appendScaled(value: result.measurement.value / executionTime, unit: "/sec");
393 break;
394
395 case QTest::FramesPerSecond:
396 case QTest::CPUTicks:
397 case QTest::WalltimeMilliseconds:
398 case QTest::WalltimeNanoseconds:
399 break; // no additional information
400 }
401
402 Q_ASSERT(result.iterations > 0);
403 buf.appendf(format: " (total: %s, iterations: %d)\n",
404 args: QTest::formatResult(number: result.measurement.value, significantDigits).constData(),
405 args: result.iterations);
406
407 outputMessage(str: buf);
408 }
409}
410
411QPlainTestLogger::QPlainTestLogger(const char *filename)
412 : QAbstractTestLogger(filename)
413{
414}
415
416QPlainTestLogger::~QPlainTestLogger() = default;
417
418void QPlainTestLogger::startLogging()
419{
420 QAbstractTestLogger::startLogging();
421
422 char buf[1024];
423 if (QTestLog::verboseLevel() < 0) {
424 std::snprintf(s: buf, maxlen: sizeof(buf), format: "Testing %s\n", QTestResult::currentTestObjectName());
425 } else {
426 std::snprintf(s: buf, maxlen: sizeof(buf),
427 format: "********* Start testing of %s *********\n"
428 "Config: Using QtTest library " QTEST_VERSION_STR
429 ", %s, %s %s\n", QTestResult::currentTestObjectName(), QLibraryInfo::build(),
430 qPrintable(QSysInfo::productType()), qPrintable(QSysInfo::productVersion()));
431 }
432 outputMessage(str: buf);
433}
434
435void QPlainTestLogger::stopLogging()
436{
437 char buf[1024];
438 const int timeMs = qRound(d: QTestLog::msecsTotalTime());
439 if (QTestLog::verboseLevel() < 0) {
440 std::snprintf(s: buf, maxlen: sizeof(buf),
441 format: "Totals: %d passed, %d failed, %d skipped, %d blacklisted, %dms\n",
442 QTestLog::passCount(), QTestLog::failCount(),
443 QTestLog::skipCount(), QTestLog::blacklistCount(), timeMs);
444 } else {
445 std::snprintf(s: buf, maxlen: sizeof(buf),
446 format: "Totals: %d passed, %d failed, %d skipped, %d blacklisted, %dms\n"
447 "********* Finished testing of %s *********\n",
448 QTestLog::passCount(), QTestLog::failCount(),
449 QTestLog::skipCount(), QTestLog::blacklistCount(), timeMs,
450 QTestResult::currentTestObjectName());
451 }
452 outputMessage(str: buf);
453
454 QAbstractTestLogger::stopLogging();
455}
456
457
458void QPlainTestLogger::enterTestFunction(const char * /*function*/)
459{
460 if (QTestLog::verboseLevel() >= 1)
461 printMessage(source: MessageSource::Other, type: QTest::ptMessageType2String(type: Info), msg: "entering");
462}
463
464void QPlainTestLogger::leaveTestFunction()
465{
466}
467
468void QPlainTestLogger::addIncident(IncidentTypes type, const char *description,
469 const char *file, int line)
470{
471 // suppress B?PASS and B?XFAIL in silent mode
472 if ((type == Pass || type == BlacklistedPass || type == XFail || type == BlacklistedXFail)
473 && QTestLog::verboseLevel() < 0)
474 return;
475
476 printMessage(source: MessageSource::Incident, type: QTest::ptIncidentType2String(type), msg: description, file, line);
477}
478
479void QPlainTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &results)
480{
481 // suppress benchmark results in silent mode
482 if (QTestLog::verboseLevel() < 0 || results.isEmpty())
483 return;
484
485 printBenchmarkResultsHeader(result: results.first());
486 printBenchmarkResults(results);
487}
488
489void QPlainTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context, const QString &message)
490{
491 QAbstractTestLogger::addMessage(type, context, message);
492}
493
494void QPlainTestLogger::addMessage(MessageTypes type, const QString &message,
495 const char *file, int line)
496{
497 // suppress non-fatal messages in silent mode
498 if (type != QFatal && QTestLog::verboseLevel() < 0)
499 return;
500
501 printMessage(source: MessageSource::Other, type: QTest::ptMessageType2String(type), qPrintable(message), file, line);
502}
503
504bool QPlainTestLogger::isRepeatSupported() const
505{
506 // The plain text logger creates unstructured reports. Such reports are not
507 // parser friendly, and are unlikely to be parsed by any test reporting
508 // tools. We can therefore allow repeated test runs with minimum risk that
509 // any parsers fails to handle repeated test names.
510 return true;
511}
512
513QT_END_NAMESPACE
514

source code of qtbase/src/testlib/qplaintestlogger.cpp