1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qtaptestlogger_p.h"
5
6#include "qtestlog_p.h"
7#include "qtestresult_p.h"
8#include "qtestassert.h"
9
10#if QT_CONFIG(regularexpression)
11# include <QtCore/qregularexpression.h>
12#endif
13
14QT_BEGIN_NAMESPACE
15
16using namespace Qt::StringLiterals;
17
18/*! \internal
19 \class QTapTestLogger
20 \inmodule QtTest
21
22 QTapTestLogger implements the Test Anything Protocol v13.
23
24 The \l{Test Anything Protocol} (TAP) is a simple plain-text interface
25 between testing code and systems for reporting and analyzing test results.
26 Since QtTest doesn't build the table for a data-driven test until the test
27 is about to be run, we don't typically know how many tests we'll run until
28 we've run them, so we put The Plan at the end, rather than the beginning.
29 We summarise the results in comments following The Plan.
30
31 \section1 YAMLish
32
33 The TAP protocol supports inclusion, after a Test Line, of a "diagnostic
34 block" in YAML, containing supporting information. We use this to package
35 other information from the test, in combination with comments to record
36 information we're unable to deliver at the same time as a test line. By
37 convention, TAP producers limit themselves to a restricted subset of YAML,
38 known as YAMLish, for maximal compatibility with TAP consumers.
39
40 YAML (see \c yaml.org for details) supports three data-types: mapping (hash
41 or dictionary), sequence (list, array or set) and scalar (string or number),
42 much as JSON does. It uses indentation to indicate levels of nesting of
43 mappings and sequences within one another. A line starting with a dash (at
44 the same level of indent as its context) introduces a list entry. A line
45 starting with a key (more indented than its context, aside from any earlier
46 keys at its level) followed by a colon introduces a mapping entry; the key
47 may be either a simple word or a quoted string (within which numeric escapes
48 may be used to indicate unicode characters). The value associated with a
49 given key, or the entry in a list, can appar after the key-colon or dasy
50 either on the same line or on a succession of subsequent lines at higher
51 indent. Thus
52
53 \code
54 - first list item
55 -
56 second: list item is a mapping
57 with:
58 - two keys
59 - the second of which is a list
60 - back in the outer list, a third item
61 \endcode
62
63 In YAMLish, the top-level structure should be a hash. The keys supported for
64 this hash, with the meanings for their values, are:
65
66 \list
67 \li \c message: free text supplying supporting details
68 \li \c severity: how bad is it ?
69 \li \c source: source describing the test, as an URL (compare file, line)
70 \li \c at: identifies the function (with file and line) performing the test
71 \li \c datetime: when the test was run (ISO 8601 or HTTP format)
72 \li \c file: source of the test as a local file-name, when appropriate
73 \li \c line: line number within the source
74 \li \c name: test name
75 \li \c extensions: sub-hash in which to store random other stuff
76 \li \c actual: what happened (a.k.a. found; contrast expected)
77 \li \c expected: what was expected (a.k.a. wanted; contrast actual)
78 \li \c display: description of the result, suitable for display
79 \li \c dump: a sub-hash of variable values when the result arose
80 \li \c error: describes the error
81 \li \c backtrace: describes the call-stack of the error
82 \endlist
83
84 In practice, only \c at, \c expected and \c actual appear to be generally
85 supported.
86
87 We can also have several messages produced within a single test, so the
88 simple \c message / \c severity pair of top-level keys does not serve us
89 well. We therefore use \c extensions with a sub-tag \c messages in which to
90 package a list of messages.
91
92 \sa QAbstractTestLogger
93*/
94
95QTapTestLogger::QTapTestLogger(const char *filename)
96 : QAbstractTestLogger(filename)
97{
98}
99
100QTapTestLogger::~QTapTestLogger() = default;
101
102void QTapTestLogger::startLogging()
103{
104 QAbstractTestLogger::startLogging();
105
106 QTestCharBuffer preamble;
107 QTest::qt_asprintf(buf: &preamble, format: "TAP version 13\n"
108 // By convention, test suite names are output as diagnostics lines
109 // This is a pretty poor convention, as consumers will then treat
110 // actual diagnostics, e.g. qDebug, as test suite names o_O
111 "# %s\n", QTestResult::currentTestObjectName());
112 outputString(msg: preamble.data());
113}
114
115void QTapTestLogger::stopLogging()
116{
117 const int total = QTestLog::totalCount();
118
119 QTestCharBuffer testPlanAndStats;
120 QTest::qt_asprintf(buf: &testPlanAndStats,
121 format: "1..%d\n" // The plan (last non-diagnostic line)
122 "# tests %d\n"
123 "# pass %d\n"
124 "# fail %d\n",
125 total, total, QTestLog::passCount(), QTestLog::failCount());
126 outputString(msg: testPlanAndStats.data());
127
128 QAbstractTestLogger::stopLogging();
129}
130
131void QTapTestLogger::enterTestFunction(const char *function)
132{
133 m_firstExpectedFail.clear();
134 Q_ASSERT(!m_gatherMessages);
135 Q_ASSERT(m_comments.isEmpty());
136 Q_ASSERT(m_messages.isEmpty());
137 m_gatherMessages = function != nullptr;
138}
139
140void QTapTestLogger::enterTestData(QTestData *data)
141{
142 m_firstExpectedFail.clear();
143 if (!m_messages.isEmpty() || !m_comments.isEmpty())
144 flushMessages();
145 m_gatherMessages = data != nullptr;
146}
147
148using namespace QTestPrivate;
149
150void QTapTestLogger::outputTestLine(bool ok, int testNumber, const QTestCharBuffer &directive)
151{
152 QTestCharBuffer testIdentifier;
153 QTestPrivate::generateTestIdentifier(identifier: &testIdentifier, parts: TestFunction | TestDataTag);
154
155 QTestCharBuffer testLine;
156 QTest::qt_asprintf(buf: &testLine, format: "%s %d - %s%s\n", ok ? "ok" : "not ok",
157 testNumber, testIdentifier.data(), directive.constData());
158
159 outputString(msg: testLine.data());
160}
161
162// The indent needs to be two spaces for maximum compatibility.
163// This matches the width of the "- " prefix on a list item's first line.
164#define YAML_INDENT " "
165
166void QTapTestLogger::outputBuffer(const QTestCharBuffer &buffer)
167{
168 auto isComment = [&buffer]() {
169 return buffer.constData()[strlen(YAML_INDENT)] == '#';
170 };
171 if (!m_gatherMessages)
172 outputString(msg: buffer.constData());
173 else
174 QTestPrivate::appendCharBuffer(accumulator: isComment() ? &m_comments : &m_messages, more: buffer);
175}
176
177void QTapTestLogger::beginYamlish()
178{
179 outputString(YAML_INDENT "---\n");
180}
181
182void QTapTestLogger::endYamlish()
183{
184 // Flush any accumulated messages:
185 if (!m_messages.isEmpty()) {
186 outputString(YAML_INDENT "extensions:\n");
187 outputString(YAML_INDENT YAML_INDENT "messages:\n");
188 outputString(msg: m_messages.constData());
189 m_messages.clear();
190 }
191 outputString(YAML_INDENT "...\n");
192}
193
194void QTapTestLogger::flushComments()
195{
196 if (!m_comments.isEmpty()) {
197 outputString(msg: m_comments.constData());
198 m_comments.clear();
199 }
200}
201
202void QTapTestLogger::flushMessages()
203{
204 /* A _data() function's messages show up here. */
205 QTestCharBuffer dataLine;
206 QTest::qt_asprintf(buf: &dataLine, format: "ok %d - %s() # Data prepared\n",
207 QTestLog::totalCount(), QTestResult::currentTestFunction());
208 outputString(msg: dataLine.constData());
209 flushComments();
210 if (!m_messages.isEmpty()) {
211 beginYamlish();
212 endYamlish();
213 }
214}
215
216void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
217 const char *file, int line)
218{
219 const bool isExpectedFail = type == XFail || type == BlacklistedXFail;
220 const bool ok = (m_firstExpectedFail.isEmpty()
221 && (type == Pass || type == BlacklistedPass || type == Skip
222 || type == XPass || type == BlacklistedXPass));
223
224 const char *const incident = [type](const char *priorXFail) {
225 switch (type) {
226 // We treat expected or blacklisted failures/passes as TODO-failures/passes,
227 // which should be treated as soft issues by consumers. Not all do though :/
228 case BlacklistedPass:
229 if (priorXFail[0] != '\0')
230 return priorXFail;
231 Q_FALLTHROUGH();
232 case XFail: case BlacklistedXFail:
233 case XPass: case BlacklistedXPass:
234 case BlacklistedFail:
235 return "TODO";
236 case Skip:
237 return "SKIP";
238 case Pass:
239 if (priorXFail[0] != '\0')
240 return priorXFail;
241 Q_FALLTHROUGH();
242 case Fail:
243 break;
244 }
245 return static_cast<const char *>(nullptr);
246 }(m_firstExpectedFail.constData());
247
248 QTestCharBuffer directive;
249 if (incident) {
250 QTest::qt_asprintf(buf: &directive, format: "%s%s%s%s",
251 isExpectedFail ? "" : " # ", incident,
252 description && description[0] ? " " : "", description);
253 }
254
255 if (!isExpectedFail) {
256 m_gatherMessages = false;
257 outputTestLine(ok, testNumber: QTestLog::totalCount(), directive);
258 } else if (m_gatherMessages && m_firstExpectedFail.isEmpty()) {
259 QTestPrivate::appendCharBuffer(accumulator: &m_firstExpectedFail, more: directive);
260 }
261 flushComments();
262
263 if (!ok || !m_messages.isEmpty()) {
264 // All failures need a diagnostics section to not confuse consumers.
265 // We also need a diagnostics section when we have messages to report.
266 if (isExpectedFail) {
267 QTestCharBuffer message;
268 if (m_gatherMessages) {
269 QTest::qt_asprintf(buf: &message, YAML_INDENT YAML_INDENT "- severity: xfail\n"
270 YAML_INDENT YAML_INDENT YAML_INDENT "message:%s\n",
271 directive.constData() + 4);
272 } else {
273 QTest::qt_asprintf(buf: &message, YAML_INDENT "# xfail:%s\n", directive.constData() + 4);
274 }
275 outputBuffer(buffer: message);
276 } else {
277 beginYamlish();
278 }
279
280 if (!isExpectedFail || m_gatherMessages) {
281 const char *indent = isExpectedFail ? YAML_INDENT YAML_INDENT YAML_INDENT : YAML_INDENT;
282 if (!ok) {
283#if QT_CONFIG(regularexpression)
284 enum class OperationType {
285 Unknown,
286 Compare, /* Plain old QCOMPARE */
287 Verify, /* QVERIFY */
288 CompareOp, /* QCOMPARE_OP */
289 };
290
291 // This is fragile, but unfortunately testlib doesn't plumb
292 // the expected and actual values to the loggers (yet).
293 static const QRegularExpression verifyRegex(
294 u"^'(?<actualexpression>.*)' returned "
295 "(?<actual>\\w+)\\. \\((?<message>.*)\\)$"_s);
296
297 static const QRegularExpression compareRegex(
298 u"^(?<message>.*)\n"
299 "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
300 "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: "
301 "(?<expected>.*)$"_s);
302
303 static const QRegularExpression compareOpRegex(
304 u"^(?<message>.*)\n"
305 "\\s*Computed\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
306 "\\s*Baseline\\s+\\((?<expectedexpresssion>.*)\\)\\s*: "
307 "(?<expected>.*)$"_s);
308
309 const QString descriptionString = QString::fromUtf8(utf8: description);
310 QRegularExpressionMatch match = verifyRegex.match(subject: descriptionString);
311
312 OperationType opType = OperationType::Unknown;
313 if (match.hasMatch())
314 opType = OperationType::Verify;
315
316 if (opType == OperationType::Unknown) {
317 match = compareRegex.match(subject: descriptionString);
318 if (match.hasMatch())
319 opType = OperationType::Compare;
320 }
321
322 if (opType == OperationType::Unknown) {
323 match = compareOpRegex.match(subject: descriptionString);
324 if (match.hasMatch())
325 opType = OperationType::CompareOp;
326 }
327
328 if (opType != OperationType::Unknown) {
329 QString message = match.captured(name: u"message");
330 QLatin1StringView comparisonType;
331 QString expected;
332 QString actual;
333 const auto parenthesize = [&match](QLatin1StringView key) -> QString {
334 return " ("_L1 % match.captured(name: key) % u')';
335 };
336 const QString actualExpression = parenthesize("actualexpression"_L1);
337
338 if (opType == OperationType::Verify) {
339 comparisonType = "QVERIFY"_L1;
340 actual = match.captured(name: u"actual").toLower() % actualExpression;
341 expected = (actual.startsWith(s: "true "_L1) ? "false"_L1 : "true"_L1)
342 % actualExpression;
343 if (message.isEmpty())
344 message = u"Verification failed"_s;
345 } else if (opType == OperationType::Compare) {
346 comparisonType = "QCOMPARE"_L1;
347 expected = match.captured(name: u"expected")
348 % parenthesize("expectedexpresssion"_L1);
349 actual = match.captured(name: u"actual") % actualExpression;
350 } else {
351 struct ComparisonInfo {
352 const char *comparisonType;
353 const char *comparisonStringOp;
354 };
355 // get a proper comparison type based on the error message
356 const auto info = [](const QString &err) -> ComparisonInfo {
357 if (err.contains(s: "different"_L1))
358 return { .comparisonType: "QCOMPARE_NE", .comparisonStringOp: "!= " };
359 else if (err.contains(s: "less than or equal to"_L1))
360 return { .comparisonType: "QCOMPARE_LE", .comparisonStringOp: "<= " };
361 else if (err.contains(s: "greater than or equal to"_L1))
362 return { .comparisonType: "QCOMPARE_GE", .comparisonStringOp: ">= " };
363 else if (err.contains(s: "less than"_L1))
364 return { .comparisonType: "QCOMPARE_LT", .comparisonStringOp: "< " };
365 else if (err.contains(s: "greater than"_L1))
366 return { .comparisonType: "QCOMPARE_GT", .comparisonStringOp: "> " };
367 else if (err.contains(s: "to be equal to"_L1))
368 return { .comparisonType: "QCOMPARE_EQ", .comparisonStringOp: "== " };
369 else
370 return { .comparisonType: "Unknown", .comparisonStringOp: "" };
371 }(message);
372 comparisonType = QLatin1StringView(info.comparisonType);
373 expected = QLatin1StringView(info.comparisonStringOp)
374 % match.captured(name: u"expected")
375 % parenthesize("expectedexpresssion"_L1);
376 actual = match.captured(name: u"actual") % actualExpression;
377 }
378
379 QTestCharBuffer diagnosticsYamlish;
380 QTest::qt_asprintf(buf: &diagnosticsYamlish,
381 format: "%stype: %s\n"
382 "%smessage: %s\n"
383 // Some consumers understand 'wanted/found', others need
384 // 'expected/actual', so be compatible with both.
385 "%swanted: %s\n"
386 "%sfound: %s\n"
387 "%sexpected: %s\n"
388 "%sactual: %s\n",
389 indent, comparisonType.latin1(),
390 indent, qPrintable(message),
391 indent, qPrintable(expected), indent, qPrintable(actual),
392 indent, qPrintable(expected), indent, qPrintable(actual)
393 );
394
395 outputBuffer(buffer: diagnosticsYamlish);
396 } else
397#endif
398 if (description && !incident) {
399 QTestCharBuffer unparsableDescription;
400 QTest::qt_asprintf(buf: &unparsableDescription, YAML_INDENT "# %s\n", description);
401 outputBuffer(buffer: unparsableDescription);
402 }
403 }
404
405 if (file) {
406 QTestCharBuffer location;
407 QTest::qt_asprintf(buf: &location,
408 // The generic 'at' key is understood by most consumers.
409 format: "%sat: %s::%s() (%s:%d)\n"
410
411 // The file and line keys are for consumers that are able
412 // to read more granular location info.
413 "%sfile: %s\n"
414 "%sline: %d\n",
415
416 indent, QTestResult::currentTestObjectName(),
417 QTestResult::currentTestFunction(),
418 file, line, indent, file, indent, line
419 );
420 outputBuffer(buffer: location);
421 }
422 }
423
424 if (!isExpectedFail)
425 endYamlish();
426 }
427}
428
429void QTapTestLogger::addMessage(MessageTypes type, const QString &message,
430 const char *file, int line)
431{
432 Q_UNUSED(file);
433 Q_UNUSED(line);
434 const char *const flavor = [type]() {
435 switch (type) {
436 case QDebug: return "debug";
437 case QInfo: return "info";
438 case QWarning: return "warning";
439 case QCritical: return "critical";
440 case QFatal: return "fatal";
441 // Handle internal messages as comments
442 case Info: return "# inform";
443 case Warn: return "# warn";
444 }
445 return "unrecognised message";
446 }();
447
448 QTestCharBuffer diagnostic;
449 if (!m_gatherMessages) {
450 QTest::qt_asprintf(buf: &diagnostic, format: "%s%s: %s\n",
451 flavor[0] == '#' ? "" : "# ",
452 flavor, qPrintable(message));
453 outputString(msg: diagnostic.constData());
454 } else if (flavor[0] == '#') {
455 QTest::qt_asprintf(buf: &diagnostic, YAML_INDENT "%s: %s\n",
456 flavor, qPrintable(message));
457 QTestPrivate::appendCharBuffer(accumulator: &m_comments, more: diagnostic);
458 } else {
459 // These shall appear in a messages: sub-block of the extensions: block,
460 // so triple-indent.
461 QTest::qt_asprintf(buf: &diagnostic, YAML_INDENT YAML_INDENT "- severity: %s\n"
462 YAML_INDENT YAML_INDENT YAML_INDENT "message: %s\n",
463 flavor, qPrintable(message));
464 QTestPrivate::appendCharBuffer(accumulator: &m_messages, more: diagnostic);
465 }
466}
467
468QT_END_NAMESPACE
469

source code of qtbase/src/testlib/qtaptestlogger.cpp