1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29#include <QDir>
30#include <QEventLoop>
31#include <QPair>
32#include <QtDebug>
33
34#include "ExitCode.h"
35
36#include "Worker.h"
37
38using namespace QPatternistSDK;
39
40const char *const Worker::m_indent = " ";
41
42Worker::Worker(QEventLoop &ev,
43 const QFileInfo &baseline,
44 const QFileInfo &result) : m_finishedCount(0)
45 , m_baselineFile(baseline)
46 , m_resultFile(result)
47 , m_eventLoop(ev)
48{
49}
50
51void Worker::list(QTextStream &out, const QString &msg, QStringList &list)
52{
53 Q_ASSERT(!msg.isEmpty());
54
55 if(list.isEmpty())
56 return;
57
58 list.sort(); /* Make it pretty, and easy to read. */
59
60 out << msg << ":\n";
61
62 const QStringList::const_iterator end(list.constEnd());
63 QStringList::const_iterator it(list.constBegin());
64
65 for(; it != end; ++it)
66 out << m_indent << qPrintable(*it) << '\n';
67}
68
69static inline int count(const ResultThreader::Hash &list, const TestResult::Status stat)
70{
71 const ResultThreader::Hash::const_iterator end(list.constEnd());
72 ResultThreader::Hash::const_iterator it(list.constBegin());
73 int result = 0;
74
75 for(; it != end; ++it)
76 {
77 if(it.value() == stat)
78 ++result;
79 }
80
81 return result;
82}
83
84void Worker::threadFinished()
85{
86 ++m_finishedCount;
87 Q_ASSERT(m_finishedCount == 1 || m_finishedCount == 2);
88
89 const ResultThreader *const handler = static_cast<ResultThreader *>(sender());
90 Q_ASSERT(handler);
91
92 switch(handler->type())
93 {
94 case ResultThreader::Baseline:
95 {
96 m_baseline = handler->result();
97 break;
98 }
99 case ResultThreader::Result:
100 m_result = handler->result();
101 }
102
103 if(m_finishedCount == 1) /* One thread's missing. */
104 return;
105
106 /* Ok, both threads have now finished, and we got their results in m_result and m_baseline. */
107
108 /* No matter how this function exits, we want to delete this Worker. */
109 deleteLater();
110
111 ResultThreader::Hash::const_iterator itA(m_result.constBegin());
112 const ResultThreader::Hash::const_iterator endA(m_result.constEnd());
113 const int baselineCount = m_baseline.count();
114 const int resultCount = m_result.count();
115
116 /* If you want useful output, change the QTextStream to use stderr. */
117 //QTextStream err(stderr);
118 QByteArray out;
119 QTextStream err(&out);
120
121 if(resultCount < baselineCount)
122 {
123 err << qPrintable(QString(QLatin1String("WARNING: Test result contains %1 reports, "
124 "but the baseline contains %2, a DECREASE "
125 "of %3 tests.\n"))
126 .arg(resultCount)
127 .arg(baselineCount)
128 .arg(resultCount - baselineCount));
129 }
130 else if(resultCount > baselineCount)
131 {
132 err << qPrintable(QString(QLatin1String("NOTE: The number of tests run is more than what "
133 "the baseline specifies. Run was %1 test cases, the "
134 "baseline specifies %2; an increase of %3 tests.\n"))
135 .arg(resultCount)
136 .arg(baselineCount)
137 .arg(resultCount - baselineCount));
138 }
139
140 for(; itA != endA; ++itA)
141 {
142 const TestResult::Status result = itA.value();
143 const TestResult::Status baseline = m_baseline.value(akey: itA.key());
144
145 if(result == baseline) /* We have no change. */
146 {
147 if(result == TestResult::NotTested)
148 m_notTested.append(t: itA.key());
149 else
150 continue;
151 }
152 else if(baseline == TestResult::Pass && result == TestResult::Fail)
153 m_unexpectedFailures.append(t: itA.key());
154 else if(baseline == TestResult::Fail && result == TestResult::Pass)
155 m_unexpectedPasses.append(t: itA.key());
156 }
157
158 list(out&: err, msg: QLatin1String("Not tested"), list&: m_notTested);
159 list(out&: err, msg: QLatin1String("Unexpected failures"), list&: m_unexpectedFailures);
160 list(out&: err, msg: QLatin1String("Unexpected passes"), list&: m_unexpectedPasses);
161
162 err << "SUMMARY:\n";
163 typedef QPair<QString, int> Info;
164 typedef QList<Info> InfoList;
165 InfoList info;
166
167 const int totFail = count(list: m_result, stat: TestResult::Fail);
168 const int totPass = count(list: m_result, stat: TestResult::Pass);
169 const int total = resultCount;
170 const int notTested = m_notTested.count();
171 const int percentage = int((static_cast<double>(totPass) / total) * 100);
172
173 Q_ASSERT_X(percentage >= 0 && percentage <= 100, Q_FUNC_INFO,
174 qPrintable(QString(QLatin1String("Percentage was: %1")).arg(percentage)));
175
176 info.append(t: Info(QLatin1String("Total"), total));
177 info.append(t: Info(QLatin1String("Failures"), totFail));
178 info.append(t: Info(QLatin1String("Passes"), totPass));
179 info.append(t: Info(QLatin1String("Not tested"), notTested));
180 info.append(t: Info(QLatin1String("Pass percentage(%)"), percentage));
181 info.append(t: Info(QLatin1String("Unexpected failures"), m_unexpectedFailures.count()));
182 info.append(t: Info(QLatin1String("Unexpected passes"), m_unexpectedPasses.count()));
183
184 const InfoList::const_iterator end(info.constEnd());
185 InfoList::const_iterator it(info.constBegin());
186
187 /* List the statistics nicely in a row with padded columns. */
188 for(; it != end; ++it)
189 {
190 const QString result((((*it).first) + QLatin1Char(':')).leftJustified(width: 22, fill: QLatin1Char(' ')));
191 err << m_indent << qPrintable(result) << (*it).second << '\n';
192 }
193
194 if(!m_unexpectedFailures.isEmpty())
195 {
196 err << "FAILURE: Regressions discovered, baseline was not updated.\n";
197 err.flush();
198 QTextStream(stderr) << out;
199 m_eventLoop.exit(returnCode: ExitCode::Regression);
200 return;
201 }
202 else if(m_unexpectedPasses.isEmpty() && baselineCount == resultCount)
203 {
204 err << "Result was identical to the baseline, baseline was not updated.\n";
205 err.flush();
206 QTextStream(stderr) << out;
207 m_eventLoop.exit(returnCode: ExitCode::Success);
208 return;
209 }
210
211 /* Ok, we got unexpected successes and no regressions: let's update the baseline. */
212
213 QFile resultFile(m_resultFile.absoluteFilePath());
214
215 /* Remove the old file, otherwise QFile::copy() will fail. */
216 QDir baselineDir(m_baselineFile.absolutePath());
217 baselineDir.remove(fileName: m_baselineFile.fileName());
218
219 if(resultFile.copy(newName: m_baselineFile.absoluteFilePath()))
220 {
221 /* Give a detailed message of what's going on. */
222 if(resultCount > baselineCount)
223 err << "More tests was run than specified in the baseline, updating the baseline.\n";
224 else
225 err << "Improvement, the baseline was updated.\n";
226
227 /* We actually flag this as an error, because the new baseline must be submitted. */
228 err.flush();
229 QTextStream(stderr) << out;
230 m_eventLoop.exit(returnCode: ExitCode::Regression);
231 return;
232 }
233 else
234 {
235 err << qPrintable(QString(QLatin1String("Encountered error when updating "
236 "the baseline: %1\n"))
237 .arg(resultFile.errorString()));
238 err.flush();
239 QTextStream(stderr) << out;
240 m_eventLoop.exit(returnCode: ExitCode::WriteError);
241 return;
242 }
243}
244
245// vim: et:ts=4:sw=4:sts=4
246

source code of qtxmlpatterns/tests/auto/xmlpatternssdk/Worker.cpp