1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29#include "qbaselinetest.h"
30#include "baselineprotocol.h"
31#include <QtCore/QDir>
32#include <QFile>
33
34#define MAXCMDLINEARGS 128
35
36namespace QBaselineTest {
37
38static char *fargv[MAXCMDLINEARGS];
39static bool simfail = false;
40static PlatformInfo customInfo;
41static bool customAutoModeSet = false;
42
43static BaselineProtocol proto;
44static bool connected = false;
45static bool triedConnecting = false;
46static bool dryRunMode = false;
47static enum { UploadMissing, UploadAll, UploadNone } baselinePolicy = UploadMissing;
48
49static QByteArray curFunction;
50static ImageItemList itemList;
51static bool gotBaselines;
52
53static QString definedTestProject;
54static QString definedTestCase;
55
56
57void handleCmdLineArgs(int *argcp, char ***argvp)
58{
59 if (!argcp || !argvp)
60 return;
61
62 bool showHelp = false;
63
64 int fargc = 0;
65 int numArgs = *argcp;
66
67 for (int i = 0; i < numArgs; i++) {
68 QByteArray arg = (*argvp)[i];
69 QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0;
70
71 if (arg == "-simfail") {
72 simfail = true;
73 } else if (arg == "-fuzzlevel") {
74 i++;
75 bool ok = false;
76 (void)nextArg.toInt(ok: &ok);
77 if (!ok) {
78 qWarning() << "-fuzzlevel requires integer parameter";
79 showHelp = true;
80 break;
81 }
82 customInfo.insert(akey: "FuzzLevel", avalue: QString::fromLatin1(str: nextArg));
83 } else if (arg == "-auto") {
84 customAutoModeSet = true;
85 customInfo.setAdHocRun(false);
86 } else if (arg == "-adhoc") {
87 customAutoModeSet = true;
88 customInfo.setAdHocRun(true);
89 } else if (arg == "-setbaselines") {
90 baselinePolicy = UploadAll;
91 } else if (arg == "-nosetbaselines") {
92 baselinePolicy = UploadNone;
93 } else if (arg == "-compareto") {
94 i++;
95 int split = qMax(a: 0, b: nextArg.indexOf(c: '='));
96 QByteArray key = nextArg.left(len: split).trimmed();
97 QByteArray value = nextArg.mid(index: split+1).trimmed();
98 if (key.isEmpty() || value.isEmpty()) {
99 qWarning() << "-compareto requires parameter of the form <key>=<value>";
100 showHelp = true;
101 break;
102 }
103 customInfo.addOverride(key, value);
104 } else {
105 if ( (arg == "-help") || (arg == "--help") )
106 showHelp = true;
107 if (fargc >= MAXCMDLINEARGS) {
108 qWarning() << "Too many command line arguments!";
109 break;
110 }
111 fargv[fargc++] = (*argvp)[i];
112 }
113 }
114 *argcp = fargc;
115 *argvp = fargv;
116
117 if (showHelp) {
118 // TBD: arrange for this to be printed *after* QTest's help
119 QTextStream out(stdout);
120 out << "\n Baseline testing (lancelot) options:\n";
121 out << " -simfail : Force an image comparison mismatch. For testing purposes.\n";
122 out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
123 out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n";
124 out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n";
125 out << " -setbaselines : Store ALL rendered images as new baselines. Forces replacement of previous baselines.\n";
126 out << " -nosetbaselines : Do not store rendered images as new baselines when previous baselines are missing.\n";
127 out << " -compareto KEY=VAL : Force comparison to baselines from a different client,\n";
128 out << " for example: -compareto QtVersion=4.8.0\n";
129 out << " Multiple -compareto client specifications may be given.\n";
130 out << "\n";
131 }
132}
133
134
135void addClientProperty(const QString& key, const QString& value)
136{
137 customInfo.insert(akey: key, avalue: value);
138}
139
140
141/*
142 If a client property script is present, run it and accept its output
143 in the form of one 'key: value' property per line
144*/
145void fetchCustomClientProperties()
146{
147 QFile file("hostinfo.txt");
148 if (!file.open(flags: QIODevice::ReadOnly | QIODevice::Text))
149 return;
150 QTextStream in(&file);
151
152 while (!in.atEnd()) {
153 QString line = in.readLine().trimmed(); // ###local8bit? utf8?
154 if (line.startsWith(c: QLatin1Char('#'))) // Ignore comments in file
155 continue;
156 QString key, val;
157 int colonPos = line.indexOf(c: ':');
158 if (colonPos > 0) {
159 key = line.left(n: colonPos).simplified().replace(before: ' ', after: '_');
160 val = line.mid(position: colonPos+1).trimmed();
161 }
162 if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value?
163 addClientProperty(key, value: val);
164 else
165 qDebug() << "Unparseable script output ignored:" << line;
166 }
167}
168
169
170bool connect(QByteArray *msg, bool *error)
171{
172 if (connected) {
173 return true;
174 }
175 else if (triedConnecting) {
176 // Avoid repeated connection attempts, to avoid the program using Timeout * #testItems seconds before giving up
177 *msg = "Not connected to baseline server.";
178 *error = true;
179 return false;
180 }
181
182 triedConnecting = true;
183 fetchCustomClientProperties();
184 // Merge the platform info set by the program with the protocols default info
185 PlatformInfo clientInfo = customInfo;
186 PlatformInfo defaultInfo = PlatformInfo::localHostInfo();
187 foreach (QString key, defaultInfo.keys()) {
188 if (!clientInfo.contains(akey: key))
189 clientInfo.insert(akey: key, avalue: defaultInfo.value(akey: key));
190 }
191 if (!customAutoModeSet)
192 clientInfo.setAdHocRun(defaultInfo.isAdHocRun());
193
194 if (!definedTestProject.isEmpty())
195 clientInfo.insert(akey: PI_Project, avalue: definedTestProject);
196
197 QString testCase = definedTestCase;
198 if (testCase.isEmpty() && QTest::testObject() && QTest::testObject()->metaObject()) {
199 //qDebug() << "Trying to Read TestCaseName from Testlib!";
200 testCase = QTest::testObject()->metaObject()->className();
201 }
202 if (testCase.isEmpty()) {
203 qWarning(msg: "QBaselineTest::connect: No test case name specified, cannot connect.");
204 return false;
205 }
206
207 if (!proto.connect(testCase, dryrun: &dryRunMode, clientInfo)) {
208 *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
209 *error = true;
210 return false;
211 }
212 connected = true;
213 return true;
214}
215
216bool disconnectFromBaselineServer()
217{
218 if (proto.disconnect()) {
219 connected = false;
220 triedConnecting = false;
221 return true;
222 }
223
224 return false;
225}
226
227bool connectToBaselineServer(QByteArray *msg, const QString &testProject, const QString &testCase)
228{
229 bool dummy;
230 QByteArray dummyMsg;
231
232 definedTestProject = testProject;
233 definedTestCase = testCase;
234
235 return connect(msg: msg ? msg : &dummyMsg, error: &dummy);
236}
237
238void setAutoMode(bool mode)
239{
240 customInfo.setAdHocRun(!mode);
241 customAutoModeSet = true;
242}
243
244void setSimFail(bool fail)
245{
246 simfail = fail;
247}
248
249
250void modifyImage(QImage *img)
251{
252 uint c0 = 0x0000ff00;
253 uint c1 = 0x0080ff00;
254 img->setPixel(x: 1,y: 1,index_or_rgb: c0);
255 img->setPixel(x: 2,y: 1,index_or_rgb: c1);
256 img->setPixel(x: 3,y: 1,index_or_rgb: c0);
257 img->setPixel(x: 1,y: 2,index_or_rgb: c1);
258 img->setPixel(x: 1,y: 3,index_or_rgb: c0);
259 img->setPixel(x: 2,y: 3,index_or_rgb: c1);
260 img->setPixel(x: 3,y: 3,index_or_rgb: c0);
261 img->setPixel(x: 1,y: 4,index_or_rgb: c1);
262 img->setPixel(x: 1,y: 5,index_or_rgb: c0);
263}
264
265
266bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error)
267{
268 ImageItem item = baseline;
269 if (simfail) {
270 // Simulate test failure by forcing image mismatch; for testing purposes
271 QImage misImg = img;
272 modifyImage(img: &misImg);
273 item.image = misImg;
274 simfail = false; // One failure is typically enough
275 } else {
276 item.image = img;
277 }
278 item.imageChecksums.clear();
279 item.imageChecksums.prepend(t: ImageItem::computeChecksum(image: item.image));
280 QByteArray srvMsg;
281 switch (baseline.status) {
282 case ImageItem::Ok:
283 break;
284 case ImageItem::IgnoreItem :
285 qDebug() << msg->constData() << "Ignored, blacklisted on server.";
286 return true;
287 break;
288 case ImageItem::BaselineNotFound:
289 if (!customInfo.overrides().isEmpty() || baselinePolicy == UploadNone) {
290 qWarning() << "Cannot compare to baseline: No such baseline found on server.";
291 return true;
292 }
293 if (proto.submitNewBaseline(item, serverMsg: &srvMsg))
294 qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded.";
295 else
296 qDebug() << msg->constData() << "Baseline not found on server. Uploading of new baseline failed:" << srvMsg;
297 return true;
298 break;
299 default:
300 qWarning() << "Unexpected reply from baseline server.";
301 return true;
302 break;
303 }
304 *error = false;
305 // The actual comparison of the given image with the baseline:
306 if (baseline.imageChecksums.contains(t: item.imageChecksums.at(i: 0))) {
307 if (!proto.submitMatch(item, serverMsg: &srvMsg))
308 qWarning() << "Failed to report image match to server:" << srvMsg;
309 return true;
310 }
311 // At this point, we have established a legitimate mismatch
312 if (baselinePolicy == UploadAll) {
313 if (proto.submitNewBaseline(item, serverMsg: &srvMsg))
314 qDebug() << msg->constData() << "Forcing new baseline; uploaded ok.";
315 else
316 qDebug() << msg->constData() << "Forcing new baseline; uploading failed:" << srvMsg;
317 return true;
318 }
319 bool fuzzyMatch = false;
320 bool res = proto.submitMismatch(item, serverMsg: &srvMsg, fuzzyMatch: &fuzzyMatch);
321 if (res && fuzzyMatch) {
322 *error = true; // To force a QSKIP/debug output; somewhat kludgy
323 *msg += srvMsg;
324 return true; // The server decides: a fuzzy match means no mismatch
325 }
326 *msg += "Mismatch. See report:\n " + srvMsg;
327 if (dryRunMode) {
328 qDebug() << "Dryrun, so ignoring" << *msg;
329 return true;
330 }
331 return false;
332}
333
334bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error, int manualdatatag)
335{
336 if (!connected && !connect(msg, error))
337 return true;
338
339 QByteArray itemName;
340 bool hasName = qstrlen(str: name);
341
342 const char *tag = QTest::currentDataTag();
343 if (qstrlen(str: tag)) {
344 itemName = tag;
345 if (hasName)
346 itemName.append(c: '_').append(s: name);
347 } else {
348 itemName = hasName ? name : "default_name";
349 }
350
351 if (manualdatatag > 0)
352 {
353 itemName.prepend(s: "_");
354 itemName.prepend(a: QByteArray::number(manualdatatag));
355 }
356
357 *msg = "Baseline check of image '" + itemName + "': ";
358
359
360 ImageItem item;
361 item.itemName = QString::fromLatin1(str: itemName);
362 item.itemChecksum = checksum;
363 item.testFunction = QString::fromLatin1(str: QTest::currentTestFunction());
364 ImageItemList list;
365 list.append(t: item);
366 if (!proto.requestBaselineChecksums(testFunction: QLatin1String(QTest::currentTestFunction()), itemList: &list) || list.isEmpty()) {
367 *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
368 *error = true;
369 return true;
370 }
371
372 return compareItem(baseline: list.at(i: 0), img, msg, error);
373}
374
375
376QTestData &newRow(const char *dataTag, quint16 checksum)
377{
378 if (QTest::currentTestFunction() != curFunction) {
379 curFunction = QTest::currentTestFunction();
380 itemList.clear();
381 gotBaselines = false;
382 }
383 ImageItem item;
384 item.itemName = QString::fromLatin1(str: dataTag);
385 item.itemChecksum = checksum;
386 item.testFunction = QString::fromLatin1(str: QTest::currentTestFunction());
387 itemList.append(t: item);
388
389 return QTest::newRow(dataTag);
390}
391
392
393bool testImage(const QImage& img, QByteArray *msg, bool *error)
394{
395 if (!connected && !connect(msg, error))
396 return true;
397
398 if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) {
399 qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()";
400 return true;
401 }
402
403 if (!gotBaselines) {
404 if (!proto.requestBaselineChecksums(testFunction: QString::fromLatin1(str: QTest::currentTestFunction()), itemList: &itemList) || itemList.isEmpty()) {
405 *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
406 *error = true;
407 return true;
408 }
409 gotBaselines = true;
410 }
411
412 QString curTag = QString::fromLatin1(str: QTest::currentDataTag());
413 ImageItemList::const_iterator it = itemList.constBegin();
414 while (it != itemList.constEnd() && it->itemName != curTag)
415 ++it;
416 if (it == itemList.constEnd()) {
417 qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag;
418 return true;
419 }
420 return compareItem(baseline: *it, img, msg, error);
421}
422
423}
424

source code of qtbase/tests/baselineserver/shared/qbaselinetest.cpp