1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29
30#include <QtCore/QCoreApplication>
31#include <QtTest/QtTest>
32
33Q_DECLARE_METATYPE(QTest::TestFailMode)
34
35class tst_ExpectFail: public QObject
36{
37 Q_OBJECT
38
39private slots:
40 void xfailAndContinue() const;
41 void xfailAndAbort() const;
42 void xfailTwice() const;
43 void xfailWithQString() const;
44 void xfailDataDrivenWithQVerify_data() const;
45 void xfailDataDrivenWithQVerify() const;
46 void xfailDataDrivenWithQCompare_data() const;
47 void xfailDataDrivenWithQCompare() const;
48 void xfailOnWrongRow_data() const;
49 void xfailOnWrongRow() const;
50 void xfailOnAnyRow_data() const;
51 void xfailOnAnyRow() const;
52 void xfailWithoutVerify_data() const;
53 void xfailWithoutVerify() const;
54 void xpass() const;
55 void xpassDataDrivenWithQVerify_data() const;
56 void xpassDataDrivenWithQVerify() const;
57 void xpassDataDrivenWithQCompare_data() const;
58 void xpassDataDrivenWithQCompare() const;
59};
60
61void tst_ExpectFail::xfailAndContinue() const
62{
63 qDebug(msg: "begin");
64 QEXPECT_FAIL("", "This should xfail", Continue);
65 QVERIFY(false);
66 qDebug(msg: "after");
67}
68
69void tst_ExpectFail::xfailAndAbort() const
70{
71 qDebug(msg: "begin");
72 QEXPECT_FAIL("", "This should xfail", Abort);
73 QVERIFY(false);
74
75 // If we get here the test did not correctly abort on the previous QVERIFY.
76 QVERIFY2(false, "This should not be reached");
77}
78
79void tst_ExpectFail::xfailTwice() const
80{
81 QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort);
82 QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is "
83 "an error and should abort this test function", Abort);
84
85 // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL.
86 QVERIFY2(false, "This should not be reached");
87}
88
89void tst_ExpectFail::xfailWithQString() const
90{
91 QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue);
92 QVERIFY(false);
93
94 int bugNo = 5;
95 QString msg("The message");
96 QEXPECT_FAIL( "", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue);
97 QVERIFY(false);
98}
99
100void tst_ExpectFail::xfailDataDrivenWithQVerify_data() const
101{
102 QTest::addColumn<bool>(name: "shouldPass");
103 QTest::addColumn<QTest::TestFailMode>(name: "failMode");
104
105 QTest::newRow(dataTag: "Pass 1") << true << QTest::Abort;
106 QTest::newRow(dataTag: "Pass 2") << true << QTest::Continue;
107 QTest::newRow(dataTag: "Abort") << false << QTest::Abort;
108 QTest::newRow(dataTag: "Continue") << false << QTest::Continue;
109}
110
111void tst_ExpectFail::xfailDataDrivenWithQVerify() const
112{
113 QFETCH(bool, shouldPass);
114 QFETCH(QTest::TestFailMode, failMode);
115
116 // You can't pass a variable as the last parameter of QEXPECT_FAIL,
117 // because the macro adds "QTest::" in front of the last parameter.
118 // That is why the following code appears to be a little strange.
119 if (!shouldPass) {
120 if (failMode == QTest::Abort)
121 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort);
122 else
123 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue);
124 }
125
126 QVERIFY(shouldPass);
127
128 // If we get here, we either expected to pass or we expected to
129 // fail and the failure mode was Continue.
130 if (!shouldPass)
131 QCOMPARE(failMode, QTest::Continue);
132}
133
134void tst_ExpectFail::xfailDataDrivenWithQCompare_data() const
135{
136 QTest::addColumn<bool>(name: "shouldPass");
137 QTest::addColumn<QTest::TestFailMode>(name: "failMode");
138
139 QTest::newRow(dataTag: "Pass 1") << true << QTest::Abort;
140 QTest::newRow(dataTag: "Pass 2") << true << QTest::Continue;
141 QTest::newRow(dataTag: "Abort") << false << QTest::Abort;
142 QTest::newRow(dataTag: "Continue") << false << QTest::Continue;
143}
144
145void tst_ExpectFail::xfailDataDrivenWithQCompare() const
146{
147 QFETCH(bool, shouldPass);
148 QFETCH(QTest::TestFailMode, failMode);
149
150 // You can't pass a variable as the last parameter of QEXPECT_FAIL,
151 // because the macro adds "QTest::" in front of the last parameter.
152 // That is why the following code appears to be a little strange.
153 if (!shouldPass) {
154 if (failMode == QTest::Abort)
155 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort);
156 else
157 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue);
158 }
159
160 QCOMPARE(1, shouldPass ? 1 : 2);
161
162 // If we get here, we either expected to pass or we expected to
163 // fail and the failure mode was Continue.
164 if (!shouldPass)
165 QCOMPARE(failMode, QTest::Continue);
166}
167
168void tst_ExpectFail::xfailOnWrongRow_data() const
169{
170 QTest::addColumn<int>(name: "dummy");
171
172 QTest::newRow(dataTag: "right row") << 0;
173}
174
175void tst_ExpectFail::xfailOnWrongRow() const
176{
177 // QEXPECT_FAIL for a row that is not the current row should be ignored.
178 QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort);
179 QVERIFY(true);
180}
181
182void tst_ExpectFail::xfailOnAnyRow_data() const
183{
184 QTest::addColumn<int>(name: "dummy");
185
186 QTest::newRow(dataTag: "first row") << 0;
187 QTest::newRow(dataTag: "second row") << 1;
188}
189
190void tst_ExpectFail::xfailOnAnyRow() const
191{
192 // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL
193 // should mean that the failure is expected for all data rows.
194 QEXPECT_FAIL("", "This test should xfail", Abort);
195 QVERIFY(false);
196}
197
198void tst_ExpectFail::xfailWithoutVerify_data() const
199{
200 QTest::addColumn<int>(name: "dummy");
201
202 QTest::newRow(dataTag: "first row") << 0;
203 QTest::newRow(dataTag: "second row") << 1;
204}
205
206void tst_ExpectFail::xfailWithoutVerify() const
207{
208 QVERIFY(true);
209 QEXPECT_FAIL("", "This expected failure should be ignored", Abort);
210}
211
212void tst_ExpectFail::xpass() const
213{
214 QEXPECT_FAIL("", "This test should xpass", Abort);
215 QVERIFY(true);
216
217 // If we get here the test did not correctly abort on the previous
218 // unexpected pass.
219 QVERIFY2(false, "This should not be reached");
220}
221
222void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const
223{
224 QTest::addColumn<bool>(name: "shouldXPass");
225
226 QTest::newRow(dataTag: "XPass") << true;
227 QTest::newRow(dataTag: "Pass") << false;
228}
229
230void tst_ExpectFail::xpassDataDrivenWithQVerify() const
231{
232 QFETCH(bool, shouldXPass);
233
234 if (shouldXPass)
235 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort);
236
237 QVERIFY(true);
238
239 // We should only get here if the test wasn't supposed to xpass.
240 QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS");
241}
242
243void tst_ExpectFail::xpassDataDrivenWithQCompare_data() const
244{
245 QTest::addColumn<bool>(name: "shouldXPass");
246
247 QTest::newRow(dataTag: "XPass") << true;
248 QTest::newRow(dataTag: "Pass") << false;
249}
250
251void tst_ExpectFail::xpassDataDrivenWithQCompare() const
252{
253 QFETCH(bool, shouldXPass);
254
255 if (shouldXPass)
256 QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort);
257
258 QCOMPARE(1, 1);
259
260 // We should only get here if the test wasn't supposed to xpass.
261 QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS");
262}
263
264QTEST_MAIN(tst_ExpectFail)
265#include "tst_expectfail.moc"
266

source code of qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp