1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/private/qtestresult_p.h>
5#include <QtCore/qglobal.h>
6#include <QtCore/qstringview.h>
7
8#include <QtTest/private/qtestlog_p.h>
9#include <QtTest/qtest.h> // toString() specializations for QStringView
10#include <QtTest/qtestdata.h>
11#include <QtTest/qtestcase.h>
12#include <QtTest/qtestassert.h>
13#include <QtTest/qtesteventloop.h>
14
15#include <stdlib.h>
16#include <stdio.h>
17#include <string.h>
18
19static const char *currentAppName = nullptr;
20
21QT_BEGIN_NAMESPACE
22
23namespace QTest
24{
25 namespace Internal {
26 static bool failed = false;
27 }
28
29 static void setFailed(bool failed)
30 {
31 static const bool fatalFailure = []() {
32 static const char * const environmentVar = "QTEST_FATAL_FAIL";
33 if (!qEnvironmentVariableIsSet(varName: environmentVar))
34 return false;
35
36 bool ok;
37 const int fatal = qEnvironmentVariableIntValue(varName: environmentVar, ok: &ok);
38 return ok && fatal;
39 }();
40
41 if (failed && fatalFailure)
42 qTerminate();
43 Internal::failed = failed;
44 }
45
46 static void resetFailed()
47 {
48 setFailed(false);
49 }
50
51 static bool hasFailed()
52 {
53 return Internal::failed;
54 }
55
56 static QTestData *currentTestData = nullptr;
57 static QTestData *currentGlobalTestData = nullptr;
58 static const char *currentTestFunc = nullptr;
59 static const char *currentTestObjectName = nullptr;
60 static bool skipCurrentTest = false;
61 static bool blacklistCurrentTest = false;
62
63 static const char *expectFailComment = nullptr;
64 static int expectFailMode = 0;
65}
66
67void QTestResult::reset()
68{
69 QTest::currentTestData = nullptr;
70 QTest::currentGlobalTestData = nullptr;
71 QTest::currentTestFunc = nullptr;
72 QTest::currentTestObjectName = nullptr;
73 QTest::resetFailed();
74
75 QTest::expectFailComment = nullptr;
76 QTest::expectFailMode = 0;
77 QTest::blacklistCurrentTest = false;
78
79 QTestLog::resetCounters();
80}
81
82void QTestResult::setBlacklistCurrentTest(bool b)
83{
84 QTest::blacklistCurrentTest = b;
85}
86
87bool QTestResult::currentTestFailed()
88{
89 return QTest::hasFailed();
90}
91
92QTestData *QTestResult::currentGlobalTestData()
93{
94 return QTest::currentGlobalTestData;
95}
96
97QTestData *QTestResult::currentTestData()
98{
99 return QTest::currentTestData;
100}
101
102void QTestResult::setCurrentGlobalTestData(QTestData *data)
103{
104 QTest::currentGlobalTestData = data;
105}
106
107void QTestResult::setCurrentTestData(QTestData *data)
108{
109 QTest::currentTestData = data;
110 QTest::resetFailed();
111 if (data)
112 QTestLog::enterTestData(data);
113}
114
115void QTestResult::setCurrentTestFunction(const char *func)
116{
117 QTest::currentTestFunc = func;
118 QTest::resetFailed();
119 if (func)
120 QTestLog::enterTestFunction(function: func);
121}
122
123static void clearExpectFail()
124{
125 QTest::expectFailMode = 0;
126 delete [] const_cast<char *>(QTest::expectFailComment);
127 QTest::expectFailComment = nullptr;
128}
129
130/*!
131 This function is called after completing each test function,
132 including test functions that are not data-driven.
133
134 For data-driven functions, this is called after each call to the test
135 function, with distinct data. Otherwise, this function is called once,
136 with currentTestData() and currentGlobalTestData() set to \nullptr.
137
138 The function is called before the test's cleanup(), if it has one.
139
140 For benchmarks, this will be called after each repeat of a function
141 (with the same data row), when the benchmarking code decides to
142 re-run one to get sufficient data.
143
144 \sa finishedCurrentTestDataCleanup()
145*/
146void QTestResult::finishedCurrentTestData()
147{
148 if (QTest::expectFailMode)
149 addFailure(message: "QEXPECT_FAIL was called without any subsequent verification statements");
150
151 clearExpectFail();
152
153 if (!QTest::hasFailed() && QTestLog::unhandledIgnoreMessages()) {
154 QTestLog::printUnhandledIgnoreMessages();
155 addFailure(message: "Not all expected messages were received");
156 }
157 QTestLog::clearIgnoreMessages();
158 QTestLog::clearFailOnWarnings();
159}
160
161/*!
162 This function is called after completing each test function,
163 including test functions that are not data-driven.
164
165 For data-driven functions, this is called after each call to the test
166 function, with distinct data. Otherwise, this function is called once,
167 with currentTestData() and currentGlobalTestData() set to \nullptr.
168
169 The function is called after the test's cleanup(), if it has one.
170
171 For benchmarks, this is called after all repeat calls to the function
172 (with a given data row).
173
174 \sa finishedCurrentTestData()
175*/
176void QTestResult::finishedCurrentTestDataCleanup()
177{
178 // If the current test hasn't failed or been skipped, then it passes.
179 if (!QTest::hasFailed() && !QTest::skipCurrentTest) {
180 if (QTest::blacklistCurrentTest)
181 QTestLog::addBPass(msg: "");
182 else
183 QTestLog::addPass(msg: "");
184 }
185
186 QTestLog::clearCurrentTestState();
187 QTest::resetFailed();
188}
189
190/*!
191 This function is called after completing each test function,
192 including test functions that are data-driven.
193
194 For data-driven functions, this is called after after all data rows
195 have been tested, and the data table has been cleared, so both
196 currentTestData() and currentGlobalTestData() will be \nullptr.
197*/
198void QTestResult::finishedCurrentTestFunction()
199{
200 QTestLog::clearCurrentTestState(); // Needed if _data() skipped.
201 QTestLog::leaveTestFunction();
202
203 QTest::currentTestFunc = nullptr;
204 QTest::resetFailed();
205}
206
207const char *QTestResult::currentTestFunction()
208{
209 return QTest::currentTestFunc;
210}
211
212const char *QTestResult::currentDataTag()
213{
214 return QTest::currentTestData ? QTest::currentTestData->dataTag() : nullptr;
215}
216
217const char *QTestResult::currentGlobalDataTag()
218{
219 return QTest::currentGlobalTestData ? QTest::currentGlobalTestData->dataTag() : nullptr;
220}
221
222static bool isExpectFailData(const char *dataIndex)
223{
224 if (!dataIndex || dataIndex[0] == '\0')
225 return true;
226 if (!QTest::currentTestData)
227 return false;
228 if (strcmp(s1: dataIndex, s2: QTest::currentTestData->dataTag()) == 0)
229 return true;
230 return false;
231}
232
233bool QTestResult::expectFail(const char *dataIndex, const char *comment,
234 QTest::TestFailMode mode, const char *file, int line)
235{
236 QTEST_ASSERT(comment);
237 QTEST_ASSERT(mode > 0);
238
239 if (!isExpectFailData(dataIndex)) {
240 delete[] comment;
241 return true; // we don't care
242 }
243
244 if (QTest::expectFailMode) {
245 delete[] comment;
246 addFailure(message: "Already expecting a fail", file, line);
247 return false;
248 }
249
250 QTest::expectFailMode = mode;
251 QTest::expectFailComment = comment;
252 return true;
253}
254
255static bool checkStatement(bool statement, const char *msg, const char *file, int line)
256{
257 if (statement) {
258 if (QTest::expectFailMode) {
259 if (QTest::blacklistCurrentTest)
260 QTestLog::addBXPass(msg, file, line);
261 else
262 QTestLog::addXPass(msg, file, line);
263
264 QTest::setFailed(true);
265 // Should B?XPass always (a) continue or (b) abort, regardless of mode ?
266 bool doContinue = (QTest::expectFailMode == QTest::Continue);
267 clearExpectFail();
268 return doContinue;
269 }
270 return true;
271 }
272
273 if (QTest::expectFailMode) {
274 if (QTest::blacklistCurrentTest)
275 QTestLog::addBXFail(msg: QTest::expectFailComment, file, line);
276 else
277 QTestLog::addXFail(msg: QTest::expectFailComment, file, line);
278 bool doContinue = (QTest::expectFailMode == QTest::Continue);
279 clearExpectFail();
280 return doContinue;
281 }
282
283 QTestResult::addFailure(message: msg, file, line);
284 return false;
285}
286
287void QTestResult::fail(const char *msg, const char *file, int line)
288{
289 checkStatement(statement: false, msg, file, line);
290}
291
292bool QTestResult::verify(bool statement, const char *statementStr,
293 const char *description, const char *file, int line)
294{
295 QTEST_ASSERT(statementStr);
296
297 char msg[1024];
298 msg[0] = '\0';
299
300 if (QTestLog::verboseLevel() >= 2) {
301 qsnprintf(str: msg, n: 1024, fmt: "QVERIFY(%s)", statementStr);
302 QTestLog::info(msg, file, line);
303 }
304
305 if (statement == !!QTest::expectFailMode) {
306 qsnprintf(str: msg, n: 1024,
307 fmt: statement ? "'%s' returned TRUE unexpectedly. (%s)" : "'%s' returned FALSE. (%s)",
308 statementStr, description ? description : "");
309 }
310
311 return checkStatement(statement, msg, file, line);
312}
313
314static const char *leftArgNameForOp(QTest::ComparisonOperation op)
315{
316 return op == QTest::ComparisonOperation::CustomCompare ? "Actual " : "Computed ";
317}
318
319static const char *rightArgNameForOp(QTest::ComparisonOperation op)
320{
321 return op == QTest::ComparisonOperation::CustomCompare ? "Expected " : "Baseline ";
322}
323
324// Overload to format failures for "const char *" - no need to strdup().
325void formatFailMessage(char *msg, size_t maxMsgLen,
326 const char *failureMsg,
327 const char *val1, const char *val2,
328 const char *actual, const char *expected,
329 QTest::ComparisonOperation op)
330{
331 size_t len1 = mbstowcs(pwcs: nullptr, s: actual, n: maxMsgLen); // Last parameter is not ignored on QNX
332 size_t len2 = mbstowcs(pwcs: nullptr, s: expected, n: maxMsgLen); // (result is never larger than this).
333 const int written = qsnprintf(str: msg, n: maxMsgLen, fmt: "%s\n", failureMsg);
334 msg += written;
335 maxMsgLen -= written;
336
337 if (val1 || val2) {
338 qsnprintf(str: msg, n: maxMsgLen, fmt: " %s(%s)%*s %s\n %s(%s)%*s %s",
339 leftArgNameForOp(op), actual, qMax(a: len1, b: len2) - len1 + 1, ":",
340 val1 ? val1 : "<null>",
341 rightArgNameForOp(op), expected, qMax(a: len1, b: len2) - len2 + 1, ":",
342 val2 ? val2 : "<null>");
343 } else {
344 // only print variable names if neither value can be represented as a string
345 qsnprintf(str: msg, n: maxMsgLen, fmt: " %s: %s\n %s: %s",
346 leftArgNameForOp(op), actual, rightArgNameForOp(op), expected);
347 }
348}
349
350// Format failures using the toString() template
351template <class Actual, class Expected>
352void formatFailMessage(char *msg, size_t maxMsgLen,
353 const char *failureMsg,
354 const Actual &val1, const Expected &val2,
355 const char *actual, const char *expected,
356 QTest::ComparisonOperation op)
357{
358 const char *val1S = QTest::toString(val1);
359 const char *val2S = QTest::toString(val2);
360
361 formatFailMessage(msg, maxMsgLen, failureMsg, val1: val1S, val2: val2S, actual, expected, op);
362
363 delete [] val1S;
364 delete [] val2S;
365}
366
367template <class Actual, class Expected>
368static bool compareHelper(bool success, const char *failureMsg,
369 const Actual &val1, const Expected &val2,
370 const char *actual, const char *expected,
371 const char *file, int line,
372 bool hasValues = true)
373{
374 const size_t maxMsgLen = 1024;
375 char msg[maxMsgLen];
376 msg[0] = '\0';
377
378 QTEST_ASSERT(expected);
379 QTEST_ASSERT(actual);
380
381 if (QTestLog::verboseLevel() >= 2) {
382 qsnprintf(str: msg, n: maxMsgLen, fmt: "QCOMPARE(%s, %s)", actual, expected);
383 QTestLog::info(msg, file, line);
384 }
385
386 if (!failureMsg)
387 failureMsg = "Compared values are not the same";
388
389 if (success) {
390 if (QTest::expectFailMode) {
391 qsnprintf(str: msg, n: maxMsgLen,
392 fmt: "QCOMPARE(%s, %s) returned TRUE unexpectedly.", actual, expected);
393 }
394 return checkStatement(statement: success, msg, file, line);
395 }
396
397
398 if (!hasValues) {
399 qsnprintf(str: msg, n: maxMsgLen, fmt: "%s", failureMsg);
400 return checkStatement(statement: success, msg, file, line);
401 }
402
403 formatFailMessage(msg, maxMsgLen, failureMsg, val1, val2, actual, expected,
404 QTest::ComparisonOperation::CustomCompare);
405
406 return checkStatement(statement: success, msg, file, line);
407}
408
409// A simplified version of compareHelper that does not use string
410// representations of the values, and prints only failureMsg when the
411// comparison fails.
412static bool compareHelper(bool success, const char *failureMsg,
413 const char *actual, const char *expected,
414 const char *file, int line)
415{
416 const size_t maxMsgLen = 1024;
417 char msg[maxMsgLen];
418 msg[0] = '\0';
419
420 QTEST_ASSERT(expected);
421 QTEST_ASSERT(actual);
422 // failureMsg can be null, if we do not use it
423 QTEST_ASSERT(success || failureMsg);
424
425 if (QTestLog::verboseLevel() >= 2) {
426 qsnprintf(str: msg, n: maxMsgLen, fmt: "QCOMPARE(%s, %s)", actual, expected);
427 QTestLog::info(msg, file, line);
428 }
429
430 if (success) {
431 if (QTest::expectFailMode) {
432 qsnprintf(str: msg, n: maxMsgLen, fmt: "QCOMPARE(%s, %s) returned TRUE unexpectedly.",
433 actual, expected);
434 }
435 return checkStatement(statement: success, msg, file, line);
436 }
437
438 return checkStatement(statement: success, msg: failureMsg, file, line);
439}
440
441bool QTestResult::compare(bool success, const char *failureMsg,
442 char *val1, char *val2,
443 const char *actual, const char *expected,
444 const char *file, int line)
445{
446 const bool result = compareHelper(success, failureMsg,
447 val1: val1 != nullptr ? val1 : "<null>",
448 val2: val2 != nullptr ? val2 : "<null>",
449 actual, expected, file, line,
450 hasValues: val1 != nullptr && val2 != nullptr);
451
452 // Our caller got these from QTest::toString()
453 delete [] val1;
454 delete [] val2;
455
456 return result;
457}
458
459bool QTestResult::compare(bool success, const char *failureMsg,
460 double val1, double val2,
461 const char *actual, const char *expected,
462 const char *file, int line)
463{
464 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
465}
466
467bool QTestResult::compare(bool success, const char *failureMsg,
468 float val1, float val2,
469 const char *actual, const char *expected,
470 const char *file, int line)
471{
472 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
473}
474
475bool QTestResult::compare(bool success, const char *failureMsg,
476 int val1, int val2,
477 const char *actual, const char *expected,
478 const char *file, int line)
479{
480 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
481}
482
483#if QT_POINTER_SIZE == 8
484bool QTestResult::compare(bool success, const char *failureMsg,
485 qsizetype val1, qsizetype val2,
486 const char *actual, const char *expected,
487 const char *file, int line)
488{
489 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
490}
491#endif // QT_POINTER_SIZE == 8
492
493bool QTestResult::compare(bool success, const char *failureMsg,
494 unsigned val1, unsigned val2,
495 const char *actual, const char *expected,
496 const char *file, int line)
497{
498 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
499}
500
501bool QTestResult::compare(bool success, const char *failureMsg,
502 QStringView val1, QStringView val2,
503 const char *actual, const char *expected,
504 const char *file, int line)
505{
506 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
507}
508
509bool QTestResult::compare(bool success, const char *failureMsg,
510 QStringView val1, const QLatin1StringView &val2,
511 const char *actual, const char *expected,
512 const char *file, int line)
513{
514 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
515}
516
517bool QTestResult::compare(bool success, const char *failureMsg,
518 const QLatin1StringView & val1, QStringView val2,
519 const char *actual, const char *expected,
520 const char *file, int line)
521{
522 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
523}
524
525// Simplified version of compare() that does not take the values, because they
526// can't be converted to strings (or the user didn't want to do that).
527bool QTestResult::compare(bool success, const char *failureMsg,
528 const char *actual, const char *expeceted,
529 const char *file, int line)
530{
531 return compareHelper(success, failureMsg, actual, expected: expeceted, file, line);
532}
533
534void QTestResult::addFailure(const char *message, const char *file, int line)
535{
536 clearExpectFail();
537 if (qApp && QThread::currentThread() == qApp->thread())
538 QTestEventLoop::instance().exitLoop();
539
540 if (QTest::blacklistCurrentTest)
541 QTestLog::addBFail(msg: message, file, line);
542 else
543 QTestLog::addFail(msg: message, file, line);
544 QTest::setFailed(true);
545}
546
547void QTestResult::addSkip(const char *message, const char *file, int line)
548{
549 clearExpectFail();
550
551 QTestLog::addSkip(msg: message, file, line);
552}
553
554void QTestResult::setCurrentTestObject(const char *name)
555{
556 QTest::currentTestObjectName = name;
557}
558
559const char *QTestResult::currentTestObjectName()
560{
561 return QTest::currentTestObjectName ? QTest::currentTestObjectName : "";
562}
563
564void QTestResult::setSkipCurrentTest(bool value)
565{
566 QTest::skipCurrentTest = value;
567}
568
569bool QTestResult::skipCurrentTest()
570{
571 return QTest::skipCurrentTest;
572}
573
574void QTestResult::setCurrentAppName(const char *appName)
575{
576 ::currentAppName = appName;
577}
578
579const char *QTestResult::currentAppName()
580{
581 return ::currentAppName;
582}
583
584static const char *macroNameForOp(QTest::ComparisonOperation op)
585{
586 using namespace QTest;
587 switch (op) {
588 case ComparisonOperation::CustomCompare:
589 return "QCOMPARE"; /* not used */
590 case ComparisonOperation::Equal:
591 return "QCOMPARE_EQ";
592 case ComparisonOperation::NotEqual:
593 return "QCOMPARE_NE";
594 case ComparisonOperation::LessThan:
595 return "QCOMPARE_LT";
596 case ComparisonOperation::LessThanOrEqual:
597 return "QCOMPARE_LE";
598 case ComparisonOperation::GreaterThan:
599 return "QCOMPARE_GT";
600 case ComparisonOperation::GreaterThanOrEqual:
601 return "QCOMPARE_GE";
602 }
603 Q_UNREACHABLE_RETURN("");
604}
605
606static const char *failureMessageForOp(QTest::ComparisonOperation op)
607{
608 using namespace QTest;
609 switch (op) {
610 case ComparisonOperation::CustomCompare:
611 return "Compared values are not the same"; /* not used */
612 case ComparisonOperation::Equal:
613 return "The computed value is expected to be equal to the baseline, but is not";
614 case ComparisonOperation::NotEqual:
615 return "The computed value is expected to be different from the baseline, but is not";
616 case ComparisonOperation::LessThan:
617 return "The computed value is expected to be less than the baseline, but is not";
618 case ComparisonOperation::LessThanOrEqual:
619 return "The computed value is expected to be less than or equal to the baseline, but is not";
620 case ComparisonOperation::GreaterThan:
621 return "The computed value is expected to be greater than the baseline, but is not";
622 case ComparisonOperation::GreaterThanOrEqual:
623 return "The computed value is expected to be greater than or equal to the baseline, but is not";
624 }
625 Q_UNREACHABLE_RETURN("");
626}
627
628bool QTestResult::reportResult(bool success, qxp::function_ref<const char *()> lhs,
629 qxp::function_ref<const char *()> rhs,
630 const char *lhsExpr, const char *rhsExpr,
631 QTest::ComparisonOperation op, const char *file, int line,
632 const char *failureMessage)
633{
634 const size_t maxMsgLen = 1024;
635 char msg[maxMsgLen];
636 msg[0] = '\0';
637
638 QTEST_ASSERT(lhsExpr);
639 QTEST_ASSERT(rhsExpr);
640
641 if (QTestLog::verboseLevel() >= 2) {
642 qsnprintf(str: msg, n: maxMsgLen, fmt: "%s(%s, %s)", macroNameForOp(op), lhsExpr, rhsExpr);
643 QTestLog::info(msg, file, line);
644 }
645
646 if (success) {
647 if (QTest::expectFailMode) {
648 qsnprintf(str: msg, n: maxMsgLen, fmt: "%s(%s, %s) returned TRUE unexpectedly.",
649 macroNameForOp(op), lhsExpr, rhsExpr);
650 }
651 return checkStatement(statement: success, msg, file, line);
652 }
653
654 const std::unique_ptr<const char[]> lhsPtr{ lhs() };
655 const std::unique_ptr<const char[]> rhsPtr{ rhs() };
656
657 if (!failureMessage)
658 failureMessage = failureMessageForOp(op);
659
660 formatFailMessage(msg, maxMsgLen, failureMsg: failureMessage, val1: lhsPtr.get(), val2: rhsPtr.get(),
661 actual: lhsExpr, expected: rhsExpr, op);
662
663 return checkStatement(statement: success, msg, file, line);
664}
665
666QT_END_NAMESPACE
667

source code of qtbase/src/testlib/qtestresult.cpp