1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <QtTest/private/qtestresult_p.h>
5#include <QtCore/qglobal.h>
6#include <QtCore/qstringview.h>
7
8#include <QtTest/private/qtestlog_p.h>
9#include <QtTest/qtest.h> // toString() specializations for QStringView
10#include <QtTest/qtestdata.h>
11#include <QtTest/qtestcase.h>
12#include <QtTest/qtestassert.h>
13#include <QtTest/qtesteventloop.h>
14
15#include <climits>
16#include <cwchar>
17#include <QtCore/q26numeric.h>
18
19#include <stdlib.h>
20#include <stdio.h>
21#include <string.h>
22
23static const char *currentAppName = nullptr;
24
25QT_BEGIN_NAMESPACE
26
27namespace QTest
28{
29 namespace Internal {
30 static bool failed = false;
31 }
32
33 static void setFailed(bool failed)
34 {
35 static const bool fatalFailure = []() {
36 static const char * const environmentVar = "QTEST_FATAL_FAIL";
37 if (!qEnvironmentVariableIsSet(varName: environmentVar))
38 return false;
39
40 bool ok;
41 const int fatal = qEnvironmentVariableIntValue(varName: environmentVar, ok: &ok);
42 return ok && fatal;
43 }();
44
45 if (failed && fatalFailure)
46 qTerminate();
47 Internal::failed = failed;
48 }
49
50 static void resetFailed()
51 {
52 setFailed(false);
53 }
54
55 static bool hasFailed()
56 {
57 return Internal::failed;
58 }
59
60 static QTestData *currentTestData = nullptr;
61 static QTestData *currentGlobalTestData = nullptr;
62 static const char *currentTestFunc = nullptr;
63 static const char *currentTestObjectName = nullptr;
64 static bool skipCurrentTest = false;
65 static bool blacklistCurrentTest = false;
66
67 static const char *expectFailComment = nullptr;
68 static int expectFailMode = 0;
69}
70
71void QTestResult::reset()
72{
73 QTest::currentTestData = nullptr;
74 QTest::currentGlobalTestData = nullptr;
75 QTest::currentTestFunc = nullptr;
76 QTest::currentTestObjectName = nullptr;
77 QTest::resetFailed();
78
79 QTest::expectFailComment = nullptr;
80 QTest::expectFailMode = 0;
81 QTest::blacklistCurrentTest = false;
82
83 QTestLog::resetCounters();
84}
85
86void QTestResult::setBlacklistCurrentTest(bool b)
87{
88 QTest::blacklistCurrentTest = b;
89}
90
91bool QTestResult::currentTestFailed()
92{
93 return QTest::hasFailed();
94}
95
96QTestData *QTestResult::currentGlobalTestData()
97{
98 return QTest::currentGlobalTestData;
99}
100
101QTestData *QTestResult::currentTestData()
102{
103 return QTest::currentTestData;
104}
105
106void QTestResult::setCurrentGlobalTestData(QTestData *data)
107{
108 QTest::currentGlobalTestData = data;
109}
110
111void QTestResult::setCurrentTestData(QTestData *data)
112{
113 QTest::currentTestData = data;
114 QTest::resetFailed();
115 if (data)
116 QTestLog::enterTestData(data);
117}
118
119void QTestResult::setCurrentTestFunction(const char *func)
120{
121 QTest::currentTestFunc = func;
122 QTest::resetFailed();
123 if (func)
124 QTestLog::enterTestFunction(function: func);
125}
126
127static void clearExpectFail()
128{
129 QTest::expectFailMode = 0;
130 delete [] const_cast<char *>(QTest::expectFailComment);
131 QTest::expectFailComment = nullptr;
132}
133
134/*!
135 This function is called after completing each test function,
136 including test functions that are not data-driven.
137
138 For data-driven functions, this is called after each call to the test
139 function, with distinct data. Otherwise, this function is called once,
140 with currentTestData() and currentGlobalTestData() set to \nullptr.
141
142 The function is called before the test's cleanup(), if it has one.
143
144 For benchmarks, this will be called after each repeat of a function
145 (with the same data row), when the benchmarking code decides to
146 re-run one to get sufficient data.
147
148 \sa finishedCurrentTestDataCleanup()
149*/
150void QTestResult::finishedCurrentTestData()
151{
152 if (QTest::expectFailMode)
153 addFailure(message: "QEXPECT_FAIL was called without any subsequent verification statements");
154
155 clearExpectFail();
156}
157
158/*!
159 This function is called after completing each test function,
160 including test functions that are not data-driven.
161
162 For data-driven functions, this is called after each call to the test
163 function, with distinct data. Otherwise, this function is called once,
164 with currentTestData() and currentGlobalTestData() set to \nullptr.
165
166 The function is called after the test's cleanup(), if it has one.
167
168 For benchmarks, this is called after all repeat calls to the function
169 (with a given data row).
170
171 \sa finishedCurrentTestData()
172*/
173void QTestResult::finishedCurrentTestDataCleanup()
174{
175 if (!QTest::hasFailed() && QTestLog::unhandledIgnoreMessages()) {
176 QTestLog::printUnhandledIgnoreMessages();
177 addFailure(message: "Not all expected messages were received");
178 }
179
180 // If the current test hasn't failed or been skipped, then it passes.
181 if (!QTest::hasFailed() && !QTest::skipCurrentTest) {
182 if (QTest::blacklistCurrentTest)
183 QTestLog::addBPass(msg: "");
184 else
185 QTestLog::addPass(msg: "");
186 }
187
188 QTestLog::clearCurrentTestState();
189 QTest::resetFailed();
190}
191
192/*!
193 This function is called after completing each test function,
194 including test functions that are data-driven.
195
196 For data-driven functions, this is called after after all data rows
197 have been tested, and the data table has been cleared, so both
198 currentTestData() and currentGlobalTestData() will be \nullptr.
199*/
200void QTestResult::finishedCurrentTestFunction()
201{
202 QTestLog::clearCurrentTestState(); // Needed if _data() skipped.
203 QTestLog::leaveTestFunction();
204
205 QTest::currentTestFunc = nullptr;
206 QTest::resetFailed();
207}
208
209const char *QTestResult::currentTestFunction()
210{
211 return QTest::currentTestFunc;
212}
213
214const char *QTestResult::currentDataTag()
215{
216 return QTest::currentTestData ? QTest::currentTestData->dataTag() : nullptr;
217}
218
219const char *QTestResult::currentGlobalDataTag()
220{
221 return QTest::currentGlobalTestData ? QTest::currentGlobalTestData->dataTag() : nullptr;
222}
223
224static bool isExpectFailData(const char *dataIndex)
225{
226 if (!dataIndex || dataIndex[0] == '\0')
227 return true;
228 if (!QTest::currentTestData)
229 return false;
230 if (strcmp(s1: dataIndex, s2: QTest::currentTestData->dataTag()) == 0)
231 return true;
232 return false;
233}
234
235bool QTestResult::expectFail(const char *dataIndex, const char *comment,
236 QTest::TestFailMode mode, const char *file, int line)
237{
238 QTEST_ASSERT(comment);
239 QTEST_ASSERT(mode > 0);
240
241 if (!isExpectFailData(dataIndex)) {
242 delete[] comment;
243 return true; // we don't care
244 }
245
246 if (QTest::expectFailMode) {
247 delete[] comment;
248 addFailure(message: "Already expecting a fail", file, line);
249 return false;
250 }
251
252 QTest::expectFailMode = mode;
253 QTest::expectFailComment = comment;
254 return true;
255}
256
257static bool checkStatement(bool statement, const char *msg, const char *file, int line)
258{
259 if (statement) {
260 if (QTest::expectFailMode) {
261 if (QTest::blacklistCurrentTest)
262 QTestLog::addBXPass(msg, file, line);
263 else
264 QTestLog::addXPass(msg, file, line);
265
266 QTest::setFailed(true);
267 // Should B?XPass always (a) continue or (b) abort, regardless of mode ?
268 bool doContinue = (QTest::expectFailMode == QTest::Continue);
269 clearExpectFail();
270 return doContinue;
271 }
272 return true;
273 }
274
275 if (QTest::expectFailMode) {
276 if (QTest::blacklistCurrentTest)
277 QTestLog::addBXFail(msg: QTest::expectFailComment, file, line);
278 else
279 QTestLog::addXFail(msg: QTest::expectFailComment, file, line);
280 bool doContinue = (QTest::expectFailMode == QTest::Continue);
281 clearExpectFail();
282 return doContinue;
283 }
284
285 QTestResult::addFailure(message: msg, file, line);
286 return false;
287}
288
289void QTestResult::fail(const char *msg, const char *file, int line)
290{
291 checkStatement(statement: false, msg, file, line);
292}
293
294// QPalette's << operator produces 1363 characters. A comparison failure
295// involving two palettes can therefore require 2726 characters, not including
296// the other output produced by QTest. Users might also have their own types
297// with large amounts of output, so use a sufficiently high value here.
298static constexpr size_t maxMsgLen = 4096;
299
300bool QTestResult::verify(bool statement, const char *statementStr,
301 const char *description, const char *file, int line)
302{
303 QTEST_ASSERT(statementStr);
304
305 Q_DECL_UNINITIALIZED char msg[maxMsgLen];
306 msg[0] = '\0';
307
308 if (QTestLog::verboseLevel() >= 2) {
309 std::snprintf(s: msg, maxlen: maxMsgLen, format: "QVERIFY(%s)", statementStr);
310 QTestLog::info(msg, file, line);
311 }
312
313 if (statement == !!QTest::expectFailMode) {
314 std::snprintf(s: msg, maxlen: maxMsgLen,
315 format: statement ? "'%s' returned TRUE unexpectedly. (%s)" : "'%s' returned FALSE. (%s)",
316 statementStr, description ? description : "");
317 }
318
319 return checkStatement(statement, msg, file, line);
320}
321
322static const char *leftArgNameForOp(QTest::ComparisonOperation op)
323{
324 return op == QTest::ComparisonOperation::CustomCompare ? "Actual " : "Computed ";
325}
326
327static const char *rightArgNameForOp(QTest::ComparisonOperation op)
328{
329 return op == QTest::ComparisonOperation::CustomCompare ? "Expected " : "Baseline ";
330}
331
332static int approx_wide_len(const char *s)
333{
334 std::mbstate_t state = {};
335 // QNX might stop at max when dst == nullptr, so pass INT_MAX,
336 // being the largest value this function will return:
337 auto r = std::mbsrtowcs(dst: nullptr, src: &s, INT_MAX, ps: &state);
338 if (r == size_t(-1)) // encoding error, fall back to strlen()
339 r = strlen(s: s); // `s` was not advanced since `dst == nullptr`
340 return q26::saturate_cast<int>(x: r);
341}
342
343// Overload to format failures for "const char *" - no need to strdup().
344static Q_DECL_COLD_FUNCTION
345void formatFailMessage(char *msg, size_t maxMsgLen,
346 const char *failureMsg,
347 const char *val1, const char *val2,
348 const char *actual, const char *expected,
349 QTest::ComparisonOperation op)
350{
351 const auto len1 = approx_wide_len(s: actual);
352 const auto len2 = approx_wide_len(s: expected);
353 const int written = std::snprintf(s: msg, maxlen: maxMsgLen, format: "%s\n", failureMsg);
354 msg += written;
355 maxMsgLen -= written;
356
357 const auto protect = [](const char *s) { return s ? s : "<null>"; };
358
359 if (val1 || val2) {
360 std::snprintf(s: msg, maxlen: maxMsgLen, format: " %s(%s)%*s %s\n %s(%s)%*s %s",
361 leftArgNameForOp(op), actual, qMax(a: len1, b: len2) - len1 + 1, ":",
362 protect(val1),
363 rightArgNameForOp(op), expected, qMax(a: len1, b: len2) - len2 + 1, ":",
364 protect(val2));
365 } else {
366 // only print variable names if neither value can be represented as a string
367 std::snprintf(s: msg, maxlen: maxMsgLen, format: " %s: %s\n %s: %s",
368 leftArgNameForOp(op), actual, rightArgNameForOp(op), expected);
369 }
370}
371
372const char *
373QTest::Internal::formatPropertyTestHelperFailure(char *msg, size_t maxMsgLen,
374 const char *actual, const char *expected,
375 const char *actualExpr, const char *expectedExpr)
376{
377 formatFailMessage(msg, maxMsgLen, failureMsg: "Comparison failed!",
378 val1: actual, val2: expected, actual: actualExpr, expected: expectedExpr,
379 op: QTest::ComparisonOperation::CustomCompare);
380 return msg;
381}
382
383// Format failures using the toString() template
384template <class Actual, class Expected>
385static Q_DECL_COLD_FUNCTION
386void formatFailMessage(char *msg, size_t maxMsgLen,
387 const char *failureMsg,
388 const Actual &val1, const Expected &val2,
389 const char *actual, const char *expected,
390 QTest::ComparisonOperation op)
391{
392 const char *val1S = QTest::toString(val1);
393 const char *val2S = QTest::toString(val2);
394
395 formatFailMessage(msg, maxMsgLen, failureMsg, val1: val1S, val2: val2S, actual, expected, op);
396
397 delete [] val1S;
398 delete [] val2S;
399}
400
401template <class Actual, class Expected>
402static bool compareHelper(bool success, const char *failureMsg,
403 const Actual &val1, const Expected &val2,
404 const char *actual, const char *expected,
405 const char *file, int line,
406 bool hasValues = true)
407{
408 Q_DECL_UNINITIALIZED char msg[maxMsgLen];
409 msg[0] = '\0';
410
411 QTEST_ASSERT(expected);
412 QTEST_ASSERT(actual);
413
414 if (QTestLog::verboseLevel() >= 2) {
415 std::snprintf(s: msg, maxlen: maxMsgLen, format: "QCOMPARE(%s, %s)", actual, expected);
416 QTestLog::info(msg, file, line);
417 }
418
419 if (!failureMsg)
420 failureMsg = "Compared values are not the same";
421
422 if (success) {
423 if (QTest::expectFailMode) {
424 std::snprintf(s: msg, maxlen: maxMsgLen,
425 format: "QCOMPARE(%s, %s) returned TRUE unexpectedly.", actual, expected);
426 }
427 return checkStatement(statement: success, msg, file, line);
428 }
429
430
431 if (!hasValues) {
432 std::snprintf(s: msg, maxlen: maxMsgLen, format: "%s", failureMsg);
433 return checkStatement(statement: success, msg, file, line);
434 }
435
436 formatFailMessage(msg, maxMsgLen, failureMsg, val1, val2, actual, expected,
437 QTest::ComparisonOperation::CustomCompare);
438
439 return checkStatement(statement: success, msg, file, line);
440}
441
442// A simplified version of compareHelper that does not use string
443// representations of the values, and prints only failureMsg when the
444// comparison fails.
445static bool compareHelper(bool success, const char *failureMsg,
446 const char *actual, const char *expected,
447 const char *file, int line)
448{
449 const size_t maxMsgLen = 1024;
450 Q_DECL_UNINITIALIZED char msg[maxMsgLen];
451 msg[0] = '\0';
452
453 QTEST_ASSERT(expected);
454 QTEST_ASSERT(actual);
455 // failureMsg can be null, if we do not use it
456 QTEST_ASSERT(success || failureMsg);
457
458 if (QTestLog::verboseLevel() >= 2) {
459 std::snprintf(s: msg, maxlen: maxMsgLen, format: "QCOMPARE(%s, %s)", actual, expected);
460 QTestLog::info(msg, file, line);
461 }
462
463 if (success) {
464 if (QTest::expectFailMode) {
465 std::snprintf(s: msg, maxlen: maxMsgLen, format: "QCOMPARE(%s, %s) returned TRUE unexpectedly.",
466 actual, expected);
467 }
468 return checkStatement(statement: success, msg, file, line);
469 }
470
471 return checkStatement(statement: success, msg: failureMsg, file, line);
472}
473
474bool QTestResult::compare(bool success, const char *failureMsg,
475 char *val1, char *val2,
476 const char *actual, const char *expected,
477 const char *file, int line)
478{
479 const bool result = compareHelper(success, failureMsg,
480 val1: val1 != nullptr ? val1 : "<null>",
481 val2: val2 != nullptr ? val2 : "<null>",
482 actual, expected, file, line,
483 hasValues: val1 != nullptr && val2 != nullptr);
484
485 // Our caller got these from QTest::toString()
486 delete [] val1;
487 delete [] val2;
488
489 return result;
490}
491
492bool QTestResult::compare(bool success, const char *failureMsg,
493 double val1, double val2,
494 const char *actual, const char *expected,
495 const char *file, int line)
496{
497 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
498}
499
500bool QTestResult::compare(bool success, const char *failureMsg,
501 float val1, float val2,
502 const char *actual, const char *expected,
503 const char *file, int line)
504{
505 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
506}
507
508bool QTestResult::compare(bool success, const char *failureMsg,
509 int val1, int val2,
510 const char *actual, const char *expected,
511 const char *file, int line)
512{
513 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
514}
515
516#if QT_POINTER_SIZE == 8
517bool QTestResult::compare(bool success, const char *failureMsg,
518 qsizetype val1, qsizetype val2,
519 const char *actual, const char *expected,
520 const char *file, int line)
521{
522 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
523}
524#endif // QT_POINTER_SIZE == 8
525
526bool QTestResult::compare(bool success, const char *failureMsg,
527 unsigned val1, unsigned val2,
528 const char *actual, const char *expected,
529 const char *file, int line)
530{
531 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
532}
533
534bool QTestResult::compare(bool success, const char *failureMsg,
535 QStringView val1, QStringView val2,
536 const char *actual, const char *expected,
537 const char *file, int line)
538{
539 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
540}
541
542bool QTestResult::compare(bool success, const char *failureMsg,
543 QStringView val1, const QLatin1StringView &val2,
544 const char *actual, const char *expected,
545 const char *file, int line)
546{
547 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
548}
549
550bool QTestResult::compare(bool success, const char *failureMsg,
551 const QLatin1StringView & val1, QStringView val2,
552 const char *actual, const char *expected,
553 const char *file, int line)
554{
555 return compareHelper(success, failureMsg, val1, val2, actual, expected, file, line);
556}
557
558// Simplified version of compare() that does not take the values, because they
559// can't be converted to strings (or the user didn't want to do that).
560bool QTestResult::compare(bool success, const char *failureMsg,
561 const char *actual, const char *expeceted,
562 const char *file, int line)
563{
564 return compareHelper(success, failureMsg, actual, expected: expeceted, file, line);
565}
566
567void QTestResult::addFailure(const char *message, const char *file, int line)
568{
569 clearExpectFail();
570 if (qApp && QThread::currentThread() == qApp->thread())
571 QTestEventLoop::instance().exitLoop();
572
573 if (QTest::blacklistCurrentTest)
574 QTestLog::addBFail(msg: message, file, line);
575 else
576 QTestLog::addFail(msg: message, file, line);
577 QTest::setFailed(true);
578}
579
580void QTestResult::addSkip(const char *message, const char *file, int line)
581{
582 clearExpectFail();
583
584 QTestLog::addSkip(msg: message, file, line);
585}
586
587void QTestResult::setCurrentTestObject(const char *name)
588{
589 QTest::currentTestObjectName = name;
590}
591
592const char *QTestResult::currentTestObjectName()
593{
594 return QTest::currentTestObjectName ? QTest::currentTestObjectName : "";
595}
596
597void QTestResult::setSkipCurrentTest(bool value)
598{
599 QTest::skipCurrentTest = value;
600}
601
602bool QTestResult::skipCurrentTest()
603{
604 return QTest::skipCurrentTest;
605}
606
607void QTestResult::setCurrentAppName(const char *appName)
608{
609 ::currentAppName = appName;
610}
611
612const char *QTestResult::currentAppName()
613{
614 return ::currentAppName;
615}
616
617static const char *macroNameForOp(QTest::ComparisonOperation op)
618{
619 using namespace QTest;
620 switch (op) {
621 case ComparisonOperation::CustomCompare:
622 return "QCOMPARE"; /* not used */
623 case ComparisonOperation::Equal:
624 return "QCOMPARE_EQ";
625 case ComparisonOperation::NotEqual:
626 return "QCOMPARE_NE";
627 case ComparisonOperation::LessThan:
628 return "QCOMPARE_LT";
629 case ComparisonOperation::LessThanOrEqual:
630 return "QCOMPARE_LE";
631 case ComparisonOperation::GreaterThan:
632 return "QCOMPARE_GT";
633 case ComparisonOperation::GreaterThanOrEqual:
634 return "QCOMPARE_GE";
635 }
636 Q_UNREACHABLE_RETURN("");
637}
638
639static const char *failureMessageForOp(QTest::ComparisonOperation op)
640{
641 using namespace QTest;
642 switch (op) {
643 case ComparisonOperation::CustomCompare:
644 return "Compared values are not the same"; /* not used */
645 case ComparisonOperation::Equal:
646 return "The computed value is expected to be equal to the baseline, but is not";
647 case ComparisonOperation::NotEqual:
648 return "The computed value is expected to be different from the baseline, but is not";
649 case ComparisonOperation::LessThan:
650 return "The computed value is expected to be less than the baseline, but is not";
651 case ComparisonOperation::LessThanOrEqual:
652 return "The computed value is expected to be less than or equal to the baseline, but is not";
653 case ComparisonOperation::GreaterThan:
654 return "The computed value is expected to be greater than the baseline, but is not";
655 case ComparisonOperation::GreaterThanOrEqual:
656 return "The computed value is expected to be greater than or equal to the baseline, but is not";
657 }
658 Q_UNREACHABLE_RETURN("");
659}
660
661bool QTestResult::reportResult(bool success, const void *lhs, const void *rhs,
662 const char *(*lhsFormatter)(const void*),
663 const char *(*rhsFormatter)(const void*),
664 const char *lhsExpr, const char *rhsExpr,
665 QTest::ComparisonOperation op, const char *file, int line,
666 const char *failureMessage)
667{
668 Q_DECL_UNINITIALIZED char msg[maxMsgLen];
669 msg[0] = '\0';
670
671 QTEST_ASSERT(lhsExpr);
672 QTEST_ASSERT(rhsExpr);
673
674 if (QTestLog::verboseLevel() >= 2) {
675 std::snprintf(s: msg, maxlen: maxMsgLen, format: "%s(%s, %s)", macroNameForOp(op), lhsExpr, rhsExpr);
676 QTestLog::info(msg, file, line);
677 }
678
679 if (success) {
680 if (QTest::expectFailMode) {
681 std::snprintf(s: msg, maxlen: maxMsgLen, format: "%s(%s, %s) returned TRUE unexpectedly.",
682 macroNameForOp(op), lhsExpr, rhsExpr);
683 }
684 return checkStatement(statement: success, msg, file, line);
685 }
686
687 const std::unique_ptr<const char[]> lhsPtr{ lhsFormatter(lhs) };
688 const std::unique_ptr<const char[]> rhsPtr{ rhsFormatter(rhs) };
689
690 if (!failureMessage)
691 failureMessage = failureMessageForOp(op);
692
693 formatFailMessage(msg, maxMsgLen, failureMsg: failureMessage, val1: lhsPtr.get(), val2: rhsPtr.get(),
694 actual: lhsExpr, expected: rhsExpr, op);
695
696 return checkStatement(statement: success, msg, file, line);
697}
698
699QT_END_NAMESPACE
700

source code of qtbase/src/testlib/qtestresult.cpp