1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "scanner.h"
5#include "logging.h"
6
7#include <QtCore/qdir.h>
8#include <QtCore/qhash.h>
9#include <QtCore/qjsonarray.h>
10#include <QtCore/qjsondocument.h>
11#include <QtCore/qjsonobject.h>
12#include <QtCore/qtextstream.h>
13#include <QtCore/qvariant.h>
14
15#include <iostream>
16
17using namespace Qt::Literals::StringLiterals;
18
19namespace Scanner {
20
21static void missingPropertyWarning(const QString &filePath, const QString &property)
22{
23 std::cerr << qPrintable(tr("File %1: Missing mandatory property '%2'.").arg(
24 QDir::toNativeSeparators(filePath), property)) << std::endl;
25}
26
27static bool validatePackage(Package &p, const QString &filePath, Checks checks, LogLevel logLevel)
28{
29 bool validPackage = true;
30
31 if (p.qtParts.isEmpty())
32 p.qtParts << u"libs"_s;
33
34 if (p.name.isEmpty()) {
35 if (p.id.startsWith(s: "chromium-"_L1)) // Ignore invalid README.chromium files
36 return false;
37
38 if (logLevel != SilentLog)
39 missingPropertyWarning(filePath, property: u"Name"_s);
40 validPackage = false;
41 }
42
43 if (p.id.isEmpty()) {
44 if (logLevel != SilentLog)
45 missingPropertyWarning(filePath, property: u"Id"_s);
46 validPackage = false;
47 } else if (!p.id.isLower() || p.id.contains(c: ' '_L1)) {
48 if (logLevel != SilentLog)
49 std::cerr << qPrintable(tr("File %1: Value of 'Id' must be in lowercase and without spaces.")
50 .arg(QDir::toNativeSeparators(filePath))) << std::endl;
51 validPackage = false;
52 }
53
54 if (p.license.isEmpty()) {
55 if (logLevel != SilentLog)
56 missingPropertyWarning(filePath, property: u"License"_s);
57 validPackage = false;
58 }
59
60 if (!p.copyright.isEmpty() && !p.copyrightFile.isEmpty()) {
61 if (logLevel != SilentLog) {
62 std::cerr << qPrintable(tr("File %1: Properties 'Copyright' and 'CopyrightFile' are "
63 "mutually exclusive.")
64 .arg(QDir::toNativeSeparators(filePath)))
65 << std::endl;
66 }
67 validPackage = false;
68 }
69
70 if (p.securityCritical && p.downloadLocation.isEmpty()) {
71 if (logLevel != SilentLog)
72 missingPropertyWarning(filePath, property: u"DownloadLocation"_s);
73 validPackage = false;
74 }
75
76 for (const QString &part : std::as_const(t&: p.qtParts)) {
77 if (part != "examples"_L1 && part != "tests"_L1
78 && part != "tools"_L1 && part != "libs"_L1) {
79
80 if (logLevel != SilentLog) {
81 std::cerr << qPrintable(tr("File %1: Property 'QtPart' contains unknown element "
82 "'%2'. Valid entries are 'examples', 'tests', 'tools' "
83 "and 'libs'.").arg(
84 QDir::toNativeSeparators(filePath), part))
85 << std::endl;
86 }
87 validPackage = false;
88 }
89 }
90
91 if (!(checks & Check::Paths))
92 return validPackage;
93
94 const QDir dir = p.path;
95 if (!dir.exists()) {
96 std::cerr << qPrintable(
97 tr("File %1: Directory '%2' does not exist.")
98 .arg(QDir::toNativeSeparators(filePath), QDir::toNativeSeparators(p.path)))
99 << std::endl;
100 validPackage = false;
101 } else {
102 for (const QString &file : std::as_const(t&: p.files)) {
103 if (!dir.exists(name: file)) {
104 if (logLevel != SilentLog) {
105 std::cerr << qPrintable(
106 tr("File %1: Path '%2' does not exist in directory '%3'.")
107 .arg(QDir::toNativeSeparators(filePath),
108 QDir::toNativeSeparators(file),
109 QDir::toNativeSeparators(p.path)))
110 << std::endl;
111 }
112 validPackage = false;
113 }
114 }
115 }
116
117 return validPackage;
118}
119
120static std::optional<QStringList> toStringList(const QJsonValue &value)
121{
122 if (!value.isArray())
123 return std::nullopt;
124 QStringList result;
125 for (const auto &iter : value.toArray()) {
126 if (iter.type() != QJsonValue::String)
127 return std::nullopt;
128 result.push_back(t: iter.toString());
129 }
130 return result;
131}
132
133static std::optional<QString> arrayToMultiLineString(const QJsonValue &value)
134{
135 if (!value.isArray())
136 return std::nullopt;
137 QString result;
138 for (const auto &iter : value.toArray()) {
139 if (iter.type() != QJsonValue::String)
140 return std::nullopt;
141 result.append(s: iter.toString());
142 result.append(s: QLatin1StringView("\n"));
143 }
144 return result;
145}
146
147// Extracts SPDX license ids from a SPDX license expression.
148// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
149static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
150{
151 const QStringList spdxOperators = {
152 u"AND"_s,
153 u"OR"_s,
154 u"WITH"_s
155 };
156
157 // Replace parentheses with spaces. We're not interested in grouping.
158 const QRegularExpression parensRegex(u"[()]"_s);
159 expression.replace(re: parensRegex, after: u" "_s);
160
161 // Split the string at space boundaries to extract tokens.
162 QStringList result;
163 for (const QString &token : expression.split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts)) {
164 if (spdxOperators.contains(str: token))
165 continue;
166
167 // Remove the unary + operator, if present.
168 if (token.endsWith(c: QLatin1Char('+')))
169 result.append(t: token.mid(position: 0, n: token.size() - 1));
170 else
171 result.append(t: token);
172 }
173 return result;
174}
175
176// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
177// Return a default-constructed QString if the directory was not found.
178static QString locateLicensesDir(const QString &packageDir)
179{
180 static const QString licensesSubDir = u"LICENSES"_s;
181 QDir dir(packageDir);
182 while (true) {
183 if (!dir.exists())
184 break;
185 if (dir.cd(dirName: licensesSubDir))
186 return dir.path();
187 if (dir.isRoot() || !dir.cdUp())
188 break;
189 }
190 return {};
191}
192
193// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
194// the specified package object.
195static bool autoDetectLicenseFiles(Package &p)
196{
197 const QString licensesDirPath = locateLicensesDir(packageDir: p.path);
198 const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(expression: p.licenseId);
199
200 bool success = true;
201 QDir licensesDir(licensesDirPath);
202 QDir licensesDirLocal = p.path;
203 for (const QString &id : licenseIds) {
204 QString fileName = id + u".txt";
205 QString fileNameLocal = u"LICENSE." + id + u".txt";
206
207 if (licensesDirLocal.exists(name: fileNameLocal)) {
208 p.licenseFiles.append(t: licensesDirLocal.filePath(fileName: fileNameLocal));
209 } else if (licensesDir.exists(name: fileName)) {
210 p.licenseFiles.append(t: licensesDir.filePath(fileName));
211 } else {
212 std::cerr << qPrintable(tr("Missing expected license file:")) << std::endl;
213 std::cerr << qPrintable(QDir::toNativeSeparators(licensesDirLocal.filePath(fileNameLocal)))
214 << std::endl;
215 if (!licensesDirPath.isEmpty()) {
216 std::cerr << qPrintable(tr("or\n %1").arg(
217 QDir::toNativeSeparators(licensesDir.filePath(fileName))))
218 << std::endl;
219 }
220 success = false;
221 }
222 }
223
224 return success;
225}
226
227// Tries to interpret a json value either as a string or an array of strings, and assigns the
228// result to outList. Returns true on success, false on failure. On failure, it also conditionally
229// prints an error.
230static bool handleStringOrStringArrayJsonKey(QStringList &outList, const QString &key,
231 QJsonValueConstRef jsonValue, const QString &filePath,
232 LogLevel logLevel)
233{
234 if (jsonValue.isArray()) {
235 auto maybeStringList = toStringList(value: jsonValue);
236 if (maybeStringList)
237 outList = maybeStringList.value();
238 } else if (jsonValue.isString()) {
239 outList.append(t: jsonValue.toString());
240 } else {
241 if (logLevel != SilentLog) {
242 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
243 "string as value of %2.").arg(
244 QDir::toNativeSeparators(filePath), key))
245 << std::endl;
246 }
247 return false;
248 }
249 return true;
250}
251
252// Transforms a JSON object into a Package object
253static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
254 Checks checks, LogLevel logLevel)
255{
256 Package p;
257 bool validPackage = true;
258 const QString directory = QFileInfo(filePath).absolutePath();
259 p.path = directory;
260
261 for (auto iter = object.constBegin(); iter != object.constEnd(); ++iter) {
262 const QString key = iter.key();
263
264 if (!iter.value().isString() && key != "QtParts"_L1 && key != "SecurityCritical"_L1
265 && key != "Files"_L1 && key != "LicenseFiles"_L1 && key != "Comment"_L1
266 && key != "Copyright"_L1 && key != "CPE"_L1 && key != "PURL"_L1) {
267 if (logLevel != SilentLog)
268 std::cerr << qPrintable(tr("File %1: Expected JSON string as value of %2.").arg(
269 QDir::toNativeSeparators(filePath), key)) << std::endl;
270 validPackage = false;
271 continue;
272 }
273 const QString value = iter.value().toString();
274 if (key == "Name"_L1) {
275 p.name = value;
276 } else if (key == "Path"_L1) {
277 p.path = QDir(directory).absoluteFilePath(fileName: value);
278 } else if (key == "Files"_L1) {
279 QJsonValueConstRef jsonValue = iter.value();
280 if (jsonValue.isArray()) {
281 auto maybeStringList = toStringList(value: jsonValue);
282 if (maybeStringList)
283 p.files = maybeStringList.value();
284 } else if (jsonValue.isString()) {
285 // Legacy format: multiple values separated by space in one string.
286 p.files = value.simplified().split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts);
287 } else {
288 if (logLevel != SilentLog) {
289 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings as value "
290 "of Files.").arg(QDir::toNativeSeparators(filePath)))
291 << std::endl;
292 validPackage = false;
293 continue;
294 }
295 }
296 } else if (key == "Comment"_L1) {
297 // Accepted purely to record details of potential interest doing
298 // updates in future. Value is an arbitrary object. Any number of
299 // Comment entries may be present: JSON doesn't require names to be
300 // unique, albeit some linters may kvetch.
301 } else if (key == "Id"_L1) {
302 p.id = value;
303 } else if (key == "Homepage"_L1) {
304 p.homepage = value;
305 } else if (key == "Version"_L1) {
306 p.version = value;
307 } else if (key == "DownloadLocation"_L1) {
308 p.downloadLocation = value;
309 } else if (key == "License"_L1) {
310 p.license = value;
311 } else if (key == "LicenseId"_L1) {
312 p.licenseId = value;
313 } else if (key == "LicenseFile"_L1) {
314 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: value));
315 } else if (key == "LicenseFiles"_L1) {
316 auto strings = toStringList(value: iter.value());
317 if (!strings) {
318 if (logLevel != SilentLog)
319 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
320 .arg(QDir::toNativeSeparators(filePath), key))
321 << std::endl;
322 validPackage = false;
323 continue;
324 }
325 const QDir dir(directory);
326 for (const auto &iter : std::as_const(t&: strings.value()))
327 p.licenseFiles.push_back(t: dir.absoluteFilePath(fileName: iter));
328 } else if (key == "Copyright"_L1) {
329 QJsonValueConstRef jsonValue = iter.value();
330 if (jsonValue.isArray()) {
331 // Array joined with new lines
332 auto maybeString = arrayToMultiLineString(value: jsonValue);
333 if (maybeString)
334 p.copyright = maybeString.value();
335 } else if (jsonValue.isString()) {
336 // Legacy format: multiple values separated by space in one string.
337 p.copyright = value;
338 } else {
339 if (logLevel != SilentLog) {
340 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
341 "string as value of %2.").arg(
342 QDir::toNativeSeparators(filePath), key)) << std::endl;
343 validPackage = false;
344 continue;
345 }
346 }
347 } else if (key == "CPE"_L1) {
348 const QJsonValueConstRef jsonValue = iter.value();
349 if (!handleStringOrStringArrayJsonKey(outList&: p.cpeList, key, jsonValue, filePath, logLevel)) {
350 validPackage = false;
351 continue;
352 }
353 } else if (key == "PURL"_L1) {
354 const QJsonValueConstRef jsonValue = iter.value();
355 if (!handleStringOrStringArrayJsonKey(outList&: p.purlList, key, jsonValue, filePath, logLevel)) {
356 validPackage = false;
357 continue;
358 }
359 } else if (key == "CopyrightFile"_L1) {
360 p.copyrightFile = QDir(directory).absoluteFilePath(fileName: value);
361 } else if (key == "PackageComment"_L1) {
362 p.packageComment = value;
363 } else if (key == "QDocModule"_L1) {
364 p.qdocModule = value;
365 } else if (key == "Description"_L1) {
366 p.description = value;
367 } else if (key == "QtUsage"_L1) {
368 p.qtUsage = value;
369 } else if (key == "SecurityCritical"_L1) {
370 if (!iter.value().isBool()) {
371 std::cerr << qPrintable(tr("File %1: Expected JSON boolean in %2.")
372 .arg(QDir::toNativeSeparators(filePath), key))
373 << std::endl;
374 validPackage = false;
375 continue;
376 }
377 p.securityCritical = iter.value().toBool();
378 } else if (key == "QtParts"_L1) {
379 auto parts = toStringList(value: iter.value());
380 if (!parts) {
381 if (logLevel != SilentLog) {
382 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
383 .arg(QDir::toNativeSeparators(filePath), key))
384 << std::endl;
385 }
386 validPackage = false;
387 continue;
388 }
389 p.qtParts = parts.value();
390 } else {
391 if (logLevel != SilentLog) {
392 std::cerr << qPrintable(tr("File %1: Unknown key %2.").arg(
393 QDir::toNativeSeparators(filePath), key)) << std::endl;
394 }
395 validPackage = false;
396 }
397 }
398
399 if (!p.copyrightFile.isEmpty()) {
400 QFile file(p.copyrightFile);
401 if (!file.open(flags: QIODevice::ReadOnly)) {
402 std::cerr << qPrintable(tr("File %1: Cannot open 'CopyrightFile' %2.\n")
403 .arg(QDir::toNativeSeparators(filePath),
404 QDir::toNativeSeparators(p.copyrightFile)));
405 validPackage = false;
406 }
407 p.copyrightFileContents = QString::fromUtf8(ba: file.readAll());
408 }
409
410 if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
411 return std::nullopt;
412
413 for (const QString &licenseFile : std::as_const(t&: p.licenseFiles)) {
414 QFile file(licenseFile);
415 if (!file.open(flags: QIODevice::ReadOnly)) {
416 if (logLevel != SilentLog) {
417 std::cerr << qPrintable(tr("File %1: Cannot open 'LicenseFile' %2.\n")
418 .arg(QDir::toNativeSeparators(filePath),
419 QDir::toNativeSeparators(licenseFile)));
420 }
421 validPackage = false;
422 }
423 p.licenseFilesContents << QString::fromUtf8(ba: file.readAll()).trimmed();
424 }
425
426 if (!validatePackage(p, filePath, checks, logLevel) || !validPackage)
427 return std::nullopt;
428
429 return p;
430}
431
432// Parses a package's details from a README.chromium file
433static Package parseChromiumFile(QFile &file, const QString &filePath, LogLevel logLevel)
434{
435 const QString directory = QFileInfo(filePath).absolutePath();
436
437 // Parse the fields in the file
438 QHash<QString, QString> fields;
439
440 QTextStream in(&file);
441 while (!in.atEnd()) {
442 QString line = in.readLine().trimmed();
443 QStringList parts = line.split(sep: u":"_s);
444
445 if (parts.size() < 2)
446 continue;
447
448 QString key = parts.at(i: 0);
449 parts.removeFirst();
450 QString value = parts.join(sep: QString()).trimmed();
451
452 fields[key] = value;
453
454 if (line == "Description:"_L1) { // special field : should handle multi-lines values
455 while (!in.atEnd()) {
456 QString line = in.readLine().trimmed();
457
458 if (line.startsWith(s: "Local Modifications:"_L1)) // Don't include this part
459 break;
460
461 fields[key] += line + u"\n"_s;
462 }
463
464 break;
465 }
466 }
467
468 // Construct the Package object
469 Package p;
470
471 QString shortName = fields.contains(key: "Short Name"_L1)
472 ? fields["Short Name"_L1]
473 : fields["Name"_L1];
474 QString version = fields[u"Version"_s];
475
476 p.id = u"chromium-"_s + shortName.toLower().replace(c: QChar::Space, after: u"-"_s);
477 p.name = fields[u"Name"_s];
478 if (version != QLatin1Char('0')) // "0" : not applicable
479 p.version = version;
480 p.license = fields[u"License"_s];
481 p.homepage = fields[u"URL"_s];
482 p.qdocModule = u"qtwebengine"_s;
483 p.qtUsage = u"Used in Qt WebEngine"_s;
484 p.description = fields[u"Description"_s].trimmed();
485 p.path = directory;
486
487 QString licenseFile = fields[u"License File"_s];
488 if (licenseFile != QString() && licenseFile != "NOT_SHIPPED"_L1) {
489 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: licenseFile));
490 } else {
491 // Look for a LICENSE or COPYING file as a fallback
492 QDir dir = directory;
493
494 dir.setNameFilters({ u"LICENSE"_s, u"COPYING"_s });
495 dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
496
497 const QFileInfoList entries = dir.entryInfoList();
498 if (!entries.empty())
499 p.licenseFiles = QStringList(entries.at(i: 0).absoluteFilePath());
500 }
501
502 // let's ignore warnings regarding Chromium files for now
503 Q_UNUSED(validatePackage(p, filePath, {}, logLevel));
504
505 return p;
506}
507
508struct CursorPosition
509{
510 int line = -1;
511 int column = -1;
512};
513
514static CursorPosition mapFromOffset(const QByteArray &content, int offset)
515{
516 CursorPosition pos{ .line: 1, .column: 1 };
517 for (int i = 0; i < content.size(); ++i) {
518 if (i == offset)
519 return pos;
520
521 if (content[i] == '\n') {
522 pos.line++;
523 pos.column = 1;
524 } else {
525 pos.column++;
526 }
527 }
528 return CursorPosition();
529}
530
531std::optional<QList<Package>> readFile(const QString &filePath, Checks checks, LogLevel logLevel)
532{
533 QList<Package> packages;
534 bool errorsFound = false;
535
536 if (logLevel == VerboseLog) {
537 std::cerr << qPrintable(tr("Reading file %1...").arg(
538 QDir::toNativeSeparators(filePath))) << std::endl;
539 }
540 QFile file(filePath);
541 if (!file.open(flags: QIODevice::ReadOnly | QIODevice::Text)) {
542 if (logLevel != SilentLog)
543 std::cerr << qPrintable(tr("Could not open file %1.").arg(
544 QDir::toNativeSeparators(file.fileName()))) << std::endl;
545 return std::nullopt;
546 }
547
548 if (filePath.endsWith(s: ".json"_L1)) {
549 QJsonParseError jsonParseError;
550 const QByteArray content = file.readAll();
551 const QJsonDocument document = QJsonDocument::fromJson(json: content, error: &jsonParseError);
552 if (document.isNull()) {
553 if (logLevel != SilentLog) {
554 const CursorPosition pos = mapFromOffset(content, offset: jsonParseError.offset);
555 std::cerr << qPrintable(tr("Could not parse file %1: %2 at line %3, column %4")
556 .arg(QDir::toNativeSeparators(file.fileName()),
557 jsonParseError.errorString(),
558 QString::number(pos.line),
559 QString::number(pos.column)))
560 << std::endl;
561 }
562 return std::nullopt;
563 }
564
565 if (document.isObject()) {
566 std::optional<Package> p =
567 readPackage(object: document.object(), filePath: file.fileName(), checks, logLevel);
568 if (p) {
569 packages << *p;
570 } else {
571 errorsFound = true;
572 }
573 } else if (document.isArray()) {
574 QJsonArray array = document.array();
575 for (int i = 0, size = array.size(); i < size; ++i) {
576 QJsonValue value = array.at(i);
577 if (value.isObject()) {
578 std::optional<Package> p =
579 readPackage(object: value.toObject(), filePath: file.fileName(), checks, logLevel);
580 if (p) {
581 packages << *p;
582 } else {
583 errorsFound = true;
584 }
585 } else {
586 if (logLevel != SilentLog) {
587 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.")
588 .arg(QDir::toNativeSeparators(file.fileName())))
589 << std::endl;
590 }
591 errorsFound = true;
592 }
593 }
594 } else {
595 if (logLevel != SilentLog) {
596 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.").arg(
597 QDir::toNativeSeparators(file.fileName()))) << std::endl;
598 }
599 errorsFound = true;
600 }
601 } else if (filePath.endsWith(s: ".chromium"_L1)) {
602 Package chromiumPackage = parseChromiumFile(file, filePath, logLevel);
603 if (!chromiumPackage.name.isEmpty()) // Skip invalid README.chromium files
604 packages << chromiumPackage;
605 } else {
606 if (logLevel != SilentLog) {
607 std::cerr << qPrintable(tr("File %1: Unsupported file type.")
608 .arg(QDir::toNativeSeparators(file.fileName())))
609 << std::endl;
610 }
611 errorsFound = true;
612 }
613
614 if (errorsFound)
615 return std::nullopt;
616 return packages;
617}
618
619std::optional<QList<Package>> scanDirectory(const QString &directory, InputFormats inputFormats,
620 Checks checks, LogLevel logLevel)
621{
622 QDir dir(directory);
623 QList<Package> packages;
624 bool errorsFound = false;
625
626 QStringList nameFilters = QStringList();
627 if (inputFormats & InputFormat::QtAttributions)
628 nameFilters << u"qt_attribution.json"_s;
629 if (inputFormats & InputFormat::ChromiumAttributions)
630 nameFilters << u"README.chromium"_s;
631 if (qEnvironmentVariableIsSet(varName: "QT_ATTRIBUTIONSSCANNER_TEST"))
632 nameFilters << u"qt_attribution_test.json"_s << u"README_test.chromium"_s;
633
634 dir.setNameFilters(nameFilters);
635 dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Files);
636
637 const QFileInfoList entries = dir.entryInfoList();
638 for (const QFileInfo &info : entries) {
639 if (info.isDir()) {
640 std::optional<QList<Package>> ps =
641 scanDirectory(directory: info.filePath(), inputFormats, checks, logLevel);
642 if (!ps)
643 errorsFound = true;
644 else
645 packages += *ps;
646 } else {
647 std::optional p = readFile(filePath: info.filePath(), checks, logLevel);
648 if (!p)
649 errorsFound = true;
650 else
651 packages += *p;
652 }
653 }
654
655 if (errorsFound)
656 return std::nullopt;
657 return packages;
658}
659
660} // namespace Scanner
661

source code of qttools/src/qtattributionsscanner/scanner.cpp