1 | /**************************************************************************** |
2 | ** |
3 | ** Copyright (C) 2016 The Qt Company Ltd. |
4 | ** Contact: https://www.qt.io/licensing/ |
5 | ** |
6 | ** This file is part of the test suite of the Qt Toolkit. |
7 | ** |
8 | ** $QT_BEGIN_LICENSE:GPL-EXCEPT$ |
9 | ** Commercial License Usage |
10 | ** Licensees holding valid commercial Qt licenses may use this file in |
11 | ** accordance with the commercial license agreement provided with the |
12 | ** Software or, alternatively, in accordance with the terms contained in |
13 | ** a written agreement between you and The Qt Company. For licensing terms |
14 | ** and conditions see https://www.qt.io/terms-conditions. For further |
15 | ** information use the contact form at https://www.qt.io/contact-us. |
16 | ** |
17 | ** GNU General Public License Usage |
18 | ** Alternatively, this file may be used under the terms of the GNU |
19 | ** General Public License version 3 as published by the Free Software |
20 | ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT |
21 | ** included in the packaging of this file. Please review the following |
22 | ** information to ensure the GNU General Public License requirements will |
23 | ** be met: https://www.gnu.org/licenses/gpl-3.0.html. |
24 | ** |
25 | ** $QT_END_LICENSE$ |
26 | ** |
27 | ****************************************************************************/ |
28 | |
29 | #include <QDir> |
30 | #include <QEventLoop> |
31 | #include <QPair> |
32 | #include <QtDebug> |
33 | |
34 | #include "ExitCode.h" |
35 | |
36 | #include "Worker.h" |
37 | |
38 | using namespace QPatternistSDK; |
39 | |
40 | const char *const Worker::m_indent = " " ; |
41 | |
42 | Worker::Worker(QEventLoop &ev, |
43 | const QFileInfo &baseline, |
44 | const QFileInfo &result) : m_finishedCount(0) |
45 | , m_baselineFile(baseline) |
46 | , m_resultFile(result) |
47 | , m_eventLoop(ev) |
48 | { |
49 | } |
50 | |
51 | void Worker::list(QTextStream &out, const QString &msg, QStringList &list) |
52 | { |
53 | Q_ASSERT(!msg.isEmpty()); |
54 | |
55 | if(list.isEmpty()) |
56 | return; |
57 | |
58 | list.sort(); /* Make it pretty, and easy to read. */ |
59 | |
60 | out << msg << ":\n" ; |
61 | |
62 | const QStringList::const_iterator end(list.constEnd()); |
63 | QStringList::const_iterator it(list.constBegin()); |
64 | |
65 | for(; it != end; ++it) |
66 | out << m_indent << qPrintable(*it) << '\n'; |
67 | } |
68 | |
69 | static inline int count(const ResultThreader::Hash &list, const TestResult::Status stat) |
70 | { |
71 | const ResultThreader::Hash::const_iterator end(list.constEnd()); |
72 | ResultThreader::Hash::const_iterator it(list.constBegin()); |
73 | int result = 0; |
74 | |
75 | for(; it != end; ++it) |
76 | { |
77 | if(it.value() == stat) |
78 | ++result; |
79 | } |
80 | |
81 | return result; |
82 | } |
83 | |
84 | void Worker::threadFinished() |
85 | { |
86 | ++m_finishedCount; |
87 | Q_ASSERT(m_finishedCount == 1 || m_finishedCount == 2); |
88 | |
89 | const ResultThreader *const handler = static_cast<ResultThreader *>(sender()); |
90 | Q_ASSERT(handler); |
91 | |
92 | switch(handler->type()) |
93 | { |
94 | case ResultThreader::Baseline: |
95 | { |
96 | m_baseline = handler->result(); |
97 | break; |
98 | } |
99 | case ResultThreader::Result: |
100 | m_result = handler->result(); |
101 | } |
102 | |
103 | if(m_finishedCount == 1) /* One thread's missing. */ |
104 | return; |
105 | |
106 | /* Ok, both threads have now finished, and we got their results in m_result and m_baseline. */ |
107 | |
108 | /* No matter how this function exits, we want to delete this Worker. */ |
109 | deleteLater(); |
110 | |
111 | ResultThreader::Hash::const_iterator itA(m_result.constBegin()); |
112 | const ResultThreader::Hash::const_iterator endA(m_result.constEnd()); |
113 | const int baselineCount = m_baseline.count(); |
114 | const int resultCount = m_result.count(); |
115 | |
116 | /* If you want useful output, change the QTextStream to use stderr. */ |
117 | //QTextStream err(stderr); |
118 | QByteArray out; |
119 | QTextStream err(&out); |
120 | |
121 | if(resultCount < baselineCount) |
122 | { |
123 | err << qPrintable(QString(QLatin1String("WARNING: Test result contains %1 reports, " |
124 | "but the baseline contains %2, a DECREASE " |
125 | "of %3 tests.\n" )) |
126 | .arg(resultCount) |
127 | .arg(baselineCount) |
128 | .arg(resultCount - baselineCount)); |
129 | } |
130 | else if(resultCount > baselineCount) |
131 | { |
132 | err << qPrintable(QString(QLatin1String("NOTE: The number of tests run is more than what " |
133 | "the baseline specifies. Run was %1 test cases, the " |
134 | "baseline specifies %2; an increase of %3 tests.\n" )) |
135 | .arg(resultCount) |
136 | .arg(baselineCount) |
137 | .arg(resultCount - baselineCount)); |
138 | } |
139 | |
140 | for(; itA != endA; ++itA) |
141 | { |
142 | const TestResult::Status result = itA.value(); |
143 | const TestResult::Status baseline = m_baseline.value(akey: itA.key()); |
144 | |
145 | if(result == baseline) /* We have no change. */ |
146 | { |
147 | if(result == TestResult::NotTested) |
148 | m_notTested.append(t: itA.key()); |
149 | else |
150 | continue; |
151 | } |
152 | else if(baseline == TestResult::Pass && result == TestResult::Fail) |
153 | m_unexpectedFailures.append(t: itA.key()); |
154 | else if(baseline == TestResult::Fail && result == TestResult::Pass) |
155 | m_unexpectedPasses.append(t: itA.key()); |
156 | } |
157 | |
158 | list(out&: err, msg: QLatin1String("Not tested" ), list&: m_notTested); |
159 | list(out&: err, msg: QLatin1String("Unexpected failures" ), list&: m_unexpectedFailures); |
160 | list(out&: err, msg: QLatin1String("Unexpected passes" ), list&: m_unexpectedPasses); |
161 | |
162 | err << "SUMMARY:\n" ; |
163 | typedef QPair<QString, int> Info; |
164 | typedef QList<Info> InfoList; |
165 | InfoList info; |
166 | |
167 | const int totFail = count(list: m_result, stat: TestResult::Fail); |
168 | const int totPass = count(list: m_result, stat: TestResult::Pass); |
169 | const int total = resultCount; |
170 | const int notTested = m_notTested.count(); |
171 | const int percentage = int((static_cast<double>(totPass) / total) * 100); |
172 | |
173 | Q_ASSERT_X(percentage >= 0 && percentage <= 100, Q_FUNC_INFO, |
174 | qPrintable(QString(QLatin1String("Percentage was: %1" )).arg(percentage))); |
175 | |
176 | info.append(t: Info(QLatin1String("Total" ), total)); |
177 | info.append(t: Info(QLatin1String("Failures" ), totFail)); |
178 | info.append(t: Info(QLatin1String("Passes" ), totPass)); |
179 | info.append(t: Info(QLatin1String("Not tested" ), notTested)); |
180 | info.append(t: Info(QLatin1String("Pass percentage(%)" ), percentage)); |
181 | info.append(t: Info(QLatin1String("Unexpected failures" ), m_unexpectedFailures.count())); |
182 | info.append(t: Info(QLatin1String("Unexpected passes" ), m_unexpectedPasses.count())); |
183 | |
184 | const InfoList::const_iterator end(info.constEnd()); |
185 | InfoList::const_iterator it(info.constBegin()); |
186 | |
187 | /* List the statistics nicely in a row with padded columns. */ |
188 | for(; it != end; ++it) |
189 | { |
190 | const QString result((((*it).first) + QLatin1Char(':')).leftJustified(width: 22, fill: QLatin1Char(' '))); |
191 | err << m_indent << qPrintable(result) << (*it).second << '\n'; |
192 | } |
193 | |
194 | if(!m_unexpectedFailures.isEmpty()) |
195 | { |
196 | err << "FAILURE: Regressions discovered, baseline was not updated.\n" ; |
197 | err.flush(); |
198 | QTextStream(stderr) << out; |
199 | m_eventLoop.exit(returnCode: ExitCode::Regression); |
200 | return; |
201 | } |
202 | else if(m_unexpectedPasses.isEmpty() && baselineCount == resultCount) |
203 | { |
204 | err << "Result was identical to the baseline, baseline was not updated.\n" ; |
205 | err.flush(); |
206 | QTextStream(stderr) << out; |
207 | m_eventLoop.exit(returnCode: ExitCode::Success); |
208 | return; |
209 | } |
210 | |
211 | /* Ok, we got unexpected successes and no regressions: let's update the baseline. */ |
212 | |
213 | QFile resultFile(m_resultFile.absoluteFilePath()); |
214 | |
215 | /* Remove the old file, otherwise QFile::copy() will fail. */ |
216 | QDir baselineDir(m_baselineFile.absolutePath()); |
217 | baselineDir.remove(fileName: m_baselineFile.fileName()); |
218 | |
219 | if(resultFile.copy(newName: m_baselineFile.absoluteFilePath())) |
220 | { |
221 | /* Give a detailed message of what's going on. */ |
222 | if(resultCount > baselineCount) |
223 | err << "More tests was run than specified in the baseline, updating the baseline.\n" ; |
224 | else |
225 | err << "Improvement, the baseline was updated.\n" ; |
226 | |
227 | /* We actually flag this as an error, because the new baseline must be submitted. */ |
228 | err.flush(); |
229 | QTextStream(stderr) << out; |
230 | m_eventLoop.exit(returnCode: ExitCode::Regression); |
231 | return; |
232 | } |
233 | else |
234 | { |
235 | err << qPrintable(QString(QLatin1String("Encountered error when updating " |
236 | "the baseline: %1\n" )) |
237 | .arg(resultFile.errorString())); |
238 | err.flush(); |
239 | QTextStream(stderr) << out; |
240 | m_eventLoop.exit(returnCode: ExitCode::WriteError); |
241 | return; |
242 | } |
243 | } |
244 | |
245 | // vim: et:ts=4:sw=4:sts=4 |
246 | |