1// Copyright (C) 2019 BogDan Vatra <bogdan@kde.org>
2// Copyright (C) 2022 The Qt Company Ltd.
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
4
5#include <QCoreApplication>
6#include <QDir>
7#include <QHash>
8#include <QRegularExpression>
9#include <QSystemSemaphore>
10#include <QXmlStreamReader>
11
12#include <algorithm>
13#include <chrono>
14#include <functional>
15#include <thread>
16
17#include <shellquote_shared.h>
18
19#ifdef Q_CC_MSVC
20#define popen _popen
21#define QT_POPEN_READ "rb"
22#define pclose _pclose
23#else
24#define QT_POPEN_READ "r"
25#endif
26
27using namespace Qt::StringLiterals;
28
29static bool checkJunit(const QByteArray &data) {
30 QXmlStreamReader reader{data};
31 while (!reader.atEnd()) {
32 reader.readNext();
33
34 if (!reader.isStartElement())
35 continue;
36
37 if (reader.name() == QStringLiteral("error"))
38 return false;
39
40 const QString type = reader.attributes().value(QStringLiteral("type")).toString();
41 if (reader.name() == QStringLiteral("failure")) {
42 if (type == QStringLiteral("fail") || type == QStringLiteral("xpass"))
43 return false;
44 }
45 }
46
47 // Fail if there's an error after reading through all the xml output
48 return !reader.hasError();
49}
50
51static bool checkTxt(const QByteArray &data) {
52 if (data.indexOf(bv: "\nFAIL! : "_L1) >= 0)
53 return false;
54 if (data.indexOf(bv: "\nXPASS : "_L1) >= 0)
55 return false;
56 // Look for "********* Finished testing of tst_QTestName *********"
57 static const QRegularExpression testTail("\\*+ +Finished testing of .+ +\\*+"_L1);
58 return testTail.match(subject: QLatin1StringView(data)).hasMatch();
59}
60
61static bool checkCsv(const QByteArray &data) {
62 // The csv format is only suitable for benchmarks,
63 // so this is not much useful to determine test failure/success.
64 // FIXME: warn the user early on about this.
65 Q_UNUSED(data);
66 return true;
67}
68
69static bool checkXml(const QByteArray &data) {
70 QXmlStreamReader reader{data};
71 while (!reader.atEnd()) {
72 reader.readNext();
73 const QString type = reader.attributes().value(QStringLiteral("type")).toString();
74 const bool isIncident = (reader.name() == QStringLiteral("Incident"));
75 if (reader.isStartElement() && isIncident) {
76 if (type == QStringLiteral("fail") || type == QStringLiteral("xpass"))
77 return false;
78 }
79 }
80
81 // Fail if there's an error after reading through all the xml output
82 return !reader.hasError();
83}
84
85static bool checkLightxml(const QByteArray &data) {
86 // lightxml intentionally skips the root element, which technically makes it
87 // not valid XML. We'll add that ourselves for the purpose of validation.
88 QByteArray newData = data;
89 newData.prepend(s: "<root>");
90 newData.append(s: "</root>");
91 return checkXml(data: newData);
92}
93
94static bool checkTeamcity(const QByteArray &data) {
95 if (data.indexOf(bv: "' message='Failure! |[Loc: ") >= 0)
96 return false;
97 const QList<QByteArray> lines = data.trimmed().split(sep: '\n');
98 if (lines.isEmpty())
99 return false;
100 return lines.last().startsWith(bv: "##teamcity[testSuiteFinished "_L1);
101}
102
103static bool checkTap(const QByteArray &data) {
104 // This will still report blacklisted fails because QTest with TAP
105 // is not putting any data about that.
106 if (data.indexOf(bv: "\nnot ok ") >= 0)
107 return false;
108
109 static const QRegularExpression testTail("ok [0-9]* - cleanupTestCase\\(\\)"_L1);
110 return testTail.match(subject: QLatin1StringView(data)).hasMatch();
111}
112
113struct Options
114{
115 bool helpRequested = false;
116 bool verbose = false;
117 bool skipAddInstallRoot = false;
118 std::chrono::seconds timeout{480}; // 8 minutes
119 QString buildPath;
120 QString adbCommand{QStringLiteral("adb")};
121 QString makeCommand;
122 QString package;
123 QString activity;
124 QStringList testArgsList;
125 QHash<QString, QString> outFiles;
126 QString testArgs;
127 QString apkPath;
128 int sdkVersion = -1;
129 int pid = -1;
130 bool showLogcatOutput = false;
131 const QHash<QString, std::function<bool(const QByteArray &)>> checkFiles = {
132 {QStringLiteral("txt"), checkTxt},
133 {QStringLiteral("csv"), checkCsv},
134 {QStringLiteral("xml"), checkXml},
135 {QStringLiteral("lightxml"), checkLightxml},
136 {QStringLiteral("xunitxml"), checkJunit},
137 {QStringLiteral("junitxml"), checkJunit},
138 {QStringLiteral("teamcity"), checkTeamcity},
139 {QStringLiteral("tap"), checkTap},
140 };
141};
142
143static Options g_options;
144
145static bool execCommand(const QString &command, QByteArray *output = nullptr, bool verbose = false)
146{
147 if (verbose)
148 fprintf(stdout, format: "Execute %s.\n", command.toUtf8().constData());
149 FILE *process = popen(command: command.toUtf8().constData(), QT_POPEN_READ);
150
151 if (!process) {
152 fprintf(stderr, format: "Cannot execute command %s.\n", qPrintable(command));
153 return false;
154 }
155 char buffer[512];
156 while (fgets(s: buffer, n: sizeof(buffer), stream: process)) {
157 if (output)
158 output->append(s: buffer);
159 if (verbose)
160 fprintf(stdout, format: "%s", buffer);
161 }
162
163 fflush(stdout);
164 fflush(stderr);
165
166 return pclose(stream: process) == 0;
167}
168
169static bool parseOptions()
170{
171 QStringList arguments = QCoreApplication::arguments();
172 int i = 1;
173 for (; i < arguments.size(); ++i) {
174 const QString &argument = arguments.at(i);
175 if (argument.compare(QStringLiteral("--adb"), cs: Qt::CaseInsensitive) == 0) {
176 if (i + 1 == arguments.size())
177 g_options.helpRequested = true;
178 else
179 g_options.adbCommand = arguments.at(i: ++i);
180 } else if (argument.compare(QStringLiteral("--path"), cs: Qt::CaseInsensitive) == 0) {
181 if (i + 1 == arguments.size())
182 g_options.helpRequested = true;
183 else
184 g_options.buildPath = arguments.at(i: ++i);
185 } else if (argument.compare(QStringLiteral("--make"), cs: Qt::CaseInsensitive) == 0) {
186 if (i + 1 == arguments.size())
187 g_options.helpRequested = true;
188 else
189 g_options.makeCommand = arguments.at(i: ++i);
190 } else if (argument.compare(QStringLiteral("--apk"), cs: Qt::CaseInsensitive) == 0) {
191 if (i + 1 == arguments.size())
192 g_options.helpRequested = true;
193 else
194 g_options.apkPath = arguments.at(i: ++i);
195 } else if (argument.compare(QStringLiteral("--activity"), cs: Qt::CaseInsensitive) == 0) {
196 if (i + 1 == arguments.size())
197 g_options.helpRequested = true;
198 else
199 g_options.activity = arguments.at(i: ++i);
200 } else if (argument.compare(QStringLiteral("--skip-install-root"), cs: Qt::CaseInsensitive) == 0) {
201 g_options.skipAddInstallRoot = true;
202 } else if (argument.compare(QStringLiteral("--show-logcat"), cs: Qt::CaseInsensitive) == 0) {
203 g_options.showLogcatOutput = true;
204 } else if (argument.compare(QStringLiteral("--timeout"), cs: Qt::CaseInsensitive) == 0) {
205 if (i + 1 == arguments.size())
206 g_options.helpRequested = true;
207 else
208 g_options.timeout = std::chrono::seconds{arguments.at(i: ++i).toInt()};
209 } else if (argument.compare(QStringLiteral("--help"), cs: Qt::CaseInsensitive) == 0) {
210 g_options.helpRequested = true;
211 } else if (argument.compare(QStringLiteral("--verbose"), cs: Qt::CaseInsensitive) == 0) {
212 g_options.verbose = true;
213 } else if (argument.compare(QStringLiteral("--"), cs: Qt::CaseInsensitive) == 0) {
214 ++i;
215 break;
216 } else {
217 g_options.testArgsList << arguments.at(i);
218 }
219 }
220 for (;i < arguments.size(); ++i)
221 g_options.testArgsList << arguments.at(i);
222
223 if (g_options.helpRequested || g_options.buildPath.isEmpty() || g_options.apkPath.isEmpty())
224 return false;
225
226 QString serial = qEnvironmentVariable(varName: "ANDROID_DEVICE_SERIAL");
227 if (!serial.isEmpty())
228 g_options.adbCommand += QStringLiteral(" -s %1").arg(a: serial);
229 return true;
230}
231
232static void printHelp()
233{
234 fprintf(stderr, format: "Syntax: %s <options> -- [TESTARGS] \n"
235 "\n"
236 " Creates an Android package in a temp directory <destination> and\n"
237 " runs it on the default emulator/device or on the one specified by\n"
238 " \"ANDROID_DEVICE_SERIAL\" environment variable.\n"
239 "\n"
240 " Mandatory arguments:\n"
241 " --path <path>: The path where androiddeployqt builds the android package.\n"
242 "\n"
243 " --apk <apk path>: The test apk path. The apk has to exist already, if it\n"
244 " does not exist the make command must be provided for building the apk.\n"
245 "\n"
246 " Optional arguments:\n"
247 " --make <make cmd>: make command, needed to install the qt library.\n"
248 " For Qt 5.14+ this can be \"make apk\".\n"
249 "\n"
250 " --adb <adb cmd>: The Android ADB command. If missing the one from\n"
251 " $PATH will be used.\n"
252 "\n"
253 " --activity <acitvity>: The Activity to run. If missing the first\n"
254 " activity from AndroidManifest.qml file will be used.\n"
255 "\n"
256 " --timeout <seconds>: Timeout to run the test. Default is 5 minutes.\n"
257 "\n"
258 " --skip-install-root: Do not append INSTALL_ROOT=... to the make command.\n"
259 "\n"
260 " --show-logcat: Print Logcat output to stdout.\n"
261 "\n"
262 " -- Arguments that will be passed to the test application.\n"
263 "\n"
264 " --verbose: Prints out information during processing.\n"
265 "\n"
266 " --help: Displays this information.\n\n",
267 qPrintable(QCoreApplication::arguments().at(0))
268 );
269}
270
271static QString packageNameFromAndroidManifest(const QString &androidManifestPath)
272{
273 QFile androidManifestXml(androidManifestPath);
274 if (androidManifestXml.open(flags: QIODevice::ReadOnly)) {
275 QXmlStreamReader reader(&androidManifestXml);
276 while (!reader.atEnd()) {
277 reader.readNext();
278 if (reader.isStartElement() && reader.name() == QStringLiteral("manifest"))
279 return reader.attributes().value(QStringLiteral("package")).toString();
280 }
281 }
282 return {};
283}
284
285static QString activityFromAndroidManifest(const QString &androidManifestPath)
286{
287 QFile androidManifestXml(androidManifestPath);
288 if (androidManifestXml.open(flags: QIODevice::ReadOnly)) {
289 QXmlStreamReader reader(&androidManifestXml);
290 while (!reader.atEnd()) {
291 reader.readNext();
292 if (reader.isStartElement() && reader.name() == QStringLiteral("activity"))
293 return reader.attributes().value(QStringLiteral("android:name")).toString();
294 }
295 }
296 return {};
297}
298
299static void setOutputFile(QString file, QString format)
300{
301 if (file.isEmpty())
302 file = QStringLiteral("-");
303 if (format.isEmpty())
304 format = QStringLiteral("txt");
305
306 g_options.outFiles[format] = file;
307}
308
309static bool parseTestArgs()
310{
311 QRegularExpression oldFormats{QStringLiteral("^-(txt|csv|xunitxml|junitxml|xml|lightxml|teamcity|tap)$")};
312 QRegularExpression newLoggingFormat{QStringLiteral("^(.*),(txt|csv|xunitxml|junitxml|xml|lightxml|teamcity|tap)$")};
313
314 QString file;
315 QString logType;
316 QStringList unhandledArgs;
317 for (int i = 0; i < g_options.testArgsList.size(); ++i) {
318 const QString &arg = g_options.testArgsList[i].trimmed();
319 if (arg == QStringLiteral("--"))
320 continue;
321 if (arg == QStringLiteral("-o")) {
322 if (i >= g_options.testArgsList.size() - 1)
323 return false; // missing file argument
324
325 const auto &filePath = g_options.testArgsList[++i];
326 const auto match = newLoggingFormat.match(subject: filePath);
327 if (!match.hasMatch()) {
328 file = filePath;
329 } else {
330 const auto capturedTexts = match.capturedTexts();
331 setOutputFile(file: capturedTexts.at(i: 1), format: capturedTexts.at(i: 2));
332 }
333 } else {
334 auto match = oldFormats.match(subject: arg);
335 if (match.hasMatch()) {
336 logType = match.capturedTexts().at(i: 1);
337 } else {
338 unhandledArgs << QStringLiteral(" \\\"%1\\\"").arg(a: arg);
339 }
340 }
341 }
342 if (g_options.outFiles.isEmpty() || !file.isEmpty() || !logType.isEmpty())
343 setOutputFile(file, format: logType);
344
345 for (const auto &format : g_options.outFiles.keys())
346 g_options.testArgs += QStringLiteral(" -o output.%1,%1").arg(a: format);
347
348 g_options.testArgs += unhandledArgs.join(sep: u' ');
349
350 g_options.testArgs = QStringLiteral("shell am start -e applicationArguments \"%1\" -n %2/%3")
351 .arg(a: shellQuote(arg: g_options.testArgs.trimmed()))
352 .arg(a: g_options.package)
353 .arg(a: g_options.activity);
354
355 return true;
356}
357
358static bool isRunning() {
359 QByteArray output;
360 if (!execCommand(QStringLiteral("%1 shell \"ps | grep ' %2'\"").arg(args&: g_options.adbCommand,
361 args: shellQuote(arg: g_options.package)), output: &output)) {
362
363 return false;
364 }
365 return output.indexOf(bv: QLatin1StringView(" " + g_options.package.toUtf8())) > -1;
366}
367
368static bool waitToFinish()
369{
370 using clock = std::chrono::system_clock;
371 auto start = clock::now();
372 // wait to start
373 while (!isRunning()) {
374 std::this_thread::sleep_for(rtime: std::chrono::milliseconds(100));
375 if ((clock::now() - start) > std::chrono::seconds{10})
376 return false;
377 }
378
379 if (g_options.sdkVersion > 23) { // pidof is broken in SDK 23, non-existent before
380 QByteArray output;
381 const QString command(QStringLiteral("%1 shell pidof -s %2")
382 .arg(args&: g_options.adbCommand, args: shellQuote(arg: g_options.package)));
383 execCommand(command, output: &output, verbose: g_options.verbose);
384 bool ok = false;
385 int pid = output.toInt(ok: &ok); // If we got more than one pid, fail.
386 if (ok) {
387 g_options.pid = pid;
388 } else {
389 fprintf(stderr,
390 format: "Unable to obtain the PID of the running unit test. Command \"%s\" "
391 "returned \"%s\"\n",
392 command.toUtf8().constData(), output.constData());
393 fflush(stderr);
394 }
395 }
396
397 // Wait to finish
398 while (isRunning()) {
399 std::this_thread::sleep_for(rtime: std::chrono::milliseconds(250));
400 if (g_options.timeout >= std::chrono::seconds::zero()
401 && (clock::now() - start) > g_options.timeout)
402 return false;
403 }
404 return true;
405}
406
407static void obtainSDKVersion()
408{
409 // SDK version is necessary, as in SDK 23 pidof is broken, so we cannot obtain the pid.
410 // Also, Logcat cannot filter by pid in SDK 23, so we don't offer the --show-logcat option.
411 QByteArray output;
412 const QString command(
413 QStringLiteral("%1 shell getprop ro.build.version.sdk").arg(a: g_options.adbCommand));
414 execCommand(command, output: &output, verbose: g_options.verbose);
415 bool ok = false;
416 int sdkVersion = output.toInt(ok: &ok);
417 if (ok) {
418 g_options.sdkVersion = sdkVersion;
419 } else {
420 fprintf(stderr,
421 format: "Unable to obtain the SDK version of the target. Command \"%s\" "
422 "returned \"%s\"\n",
423 command.toUtf8().constData(), output.constData());
424 fflush(stderr);
425 }
426}
427
428static bool pullFiles()
429{
430 bool ret = true;
431 for (auto it = g_options.outFiles.constBegin(); it != g_options.outFiles.end(); ++it) {
432 // Get only stdout from cat and get rid of stderr and fail later if the output is empty
433 const QString catCmd = QStringLiteral("cat files/output.%1 2> /dev/null").arg(a: it.key());
434
435 QByteArray output;
436 if (!execCommand(QStringLiteral("%1 shell 'run-as %2 %3'")
437 .arg(args&: g_options.adbCommand, args&: g_options.package, args: catCmd), output: &output)) {
438 // Cannot find output file. Check in path related to current user
439 QByteArray userId;
440 execCommand(QStringLiteral("%1 shell cmd activity get-current-user")
441 .arg(a: g_options.adbCommand), output: &userId);
442 const QString userIdSimplified(QString::fromUtf8(ba: userId).simplified());
443 if (!execCommand(QStringLiteral("%1 shell 'run-as %2 --user %3 %4'")
444 .arg(args&: g_options.adbCommand, args&: g_options.package, args: userIdSimplified, args: catCmd),
445 output: &output)) {
446 return false;
447 }
448 }
449
450 if (output.isEmpty()) {
451 fprintf(stderr, format: "Failed to get the test output from the target. Either the output "
452 "is empty or androidtestrunner failed to retrieve it.\n");
453 return false;
454 }
455
456 auto checkerIt = g_options.checkFiles.find(key: it.key());
457 ret = ret && checkerIt != g_options.checkFiles.end() && checkerIt.value()(output);
458 if (it.value() == QStringLiteral("-")){
459 fprintf(stdout, format: "%s", output.constData());
460 fflush(stdout);
461 } else {
462 QFile out{it.value()};
463 if (!out.open(flags: QIODevice::WriteOnly))
464 return false;
465 out.write(data: output);
466 }
467 }
468 return ret;
469}
470
471struct RunnerLocker
472{
473 RunnerLocker()
474 {
475 runner.acquire();
476 }
477 ~RunnerLocker()
478 {
479 runner.release();
480 }
481 QSystemSemaphore runner{ QSystemSemaphore::platformSafeKey(key: u"androidtestrunner"_s),
482 1, QSystemSemaphore::Open };
483};
484
485int main(int argc, char *argv[])
486{
487 QCoreApplication a(argc, argv);
488 if (!parseOptions()) {
489 printHelp();
490 return 1;
491 }
492
493 if (g_options.makeCommand.isEmpty()) {
494 fprintf(stderr,
495 format: "It is required to provide a make command with the \"--make\" parameter "
496 "to generate the apk.\n");
497 return 1;
498 }
499 if (!execCommand(command: g_options.makeCommand, output: nullptr, verbose: true)) {
500 if (!g_options.skipAddInstallRoot) {
501 // we need to run make INSTALL_ROOT=path install to install the application file(s) first
502 if (!execCommand(QStringLiteral("%1 INSTALL_ROOT=%2 install")
503 .arg(args&: g_options.makeCommand, args: QDir::toNativeSeparators(pathName: g_options.buildPath)), output: nullptr, verbose: g_options.verbose)) {
504 return 1;
505 }
506 } else {
507 if (!execCommand(QStringLiteral("%1")
508 .arg(a: g_options.makeCommand), output: nullptr, verbose: g_options.verbose)) {
509 return 1;
510 }
511 }
512 }
513
514 if (!QFile::exists(fileName: g_options.apkPath)) {
515 fprintf(stderr,
516 format: "No apk \"%s\" found after running the make command. Check the provided path and "
517 "the make command.\n",
518 qPrintable(g_options.apkPath));
519 return 1;
520 }
521
522 obtainSDKVersion();
523
524 RunnerLocker lock; // do not install or run packages while another test is running
525 if (!execCommand(QStringLiteral("%1 install -r -g %2")
526 .arg(args&: g_options.adbCommand, args&: g_options.apkPath), output: nullptr, verbose: g_options.verbose)) {
527 return 1;
528 }
529
530 QString manifest = g_options.buildPath + QStringLiteral("/AndroidManifest.xml");
531 g_options.package = packageNameFromAndroidManifest(androidManifestPath: manifest);
532 if (g_options.activity.isEmpty())
533 g_options.activity = activityFromAndroidManifest(androidManifestPath: manifest);
534
535 // parseTestArgs depends on g_options.package
536 if (!parseTestArgs())
537 return 1;
538
539 // start the tests
540 bool res = execCommand(QStringLiteral("%1 %2").arg(args&: g_options.adbCommand, args&: g_options.testArgs),
541 output: nullptr, verbose: g_options.verbose)
542 && waitToFinish();
543
544 // get logcat output
545 if (res && g_options.showLogcatOutput) {
546 if (g_options.sdkVersion <= 23) {
547 fprintf(stderr, format: "Cannot show logcat output on Android 23 and below.\n");
548 fflush(stderr);
549 } else if (g_options.pid > 0) {
550 fprintf(stdout, format: "Logcat output:\n");
551 res &= execCommand(QStringLiteral("%1 logcat -d --pid=%2")
552 .arg(a: g_options.adbCommand)
553 .arg(a: g_options.pid),
554 output: nullptr, verbose: true);
555 fprintf(stdout, format: "End Logcat output.\n");
556 }
557 }
558
559 if (res)
560 res &= pullFiles();
561 res &= execCommand(QStringLiteral("%1 uninstall %2").arg(args&: g_options.adbCommand, args&: g_options.package),
562 output: nullptr, verbose: g_options.verbose);
563 fflush(stdout);
564 return res ? 0 : 1;
565}
566

source code of qtbase/src/tools/androidtestrunner/main.cpp