1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "scanner.h"
5#include "logging.h"
6
7#include <QtCore/qdir.h>
8#include <QtCore/qhash.h>
9#include <QtCore/qjsonarray.h>
10#include <QtCore/qjsondocument.h>
11#include <QtCore/qjsonobject.h>
12#include <QtCore/qtextstream.h>
13#include <QtCore/qvariant.h>
14
15#include <iostream>
16
17using namespace Qt::Literals::StringLiterals;
18
19namespace Scanner {
20
21static void missingPropertyWarning(const QString &filePath, const QString &property)
22{
23 std::cerr << qPrintable(tr("File %1: Missing mandatory property '%2'.").arg(
24 QDir::toNativeSeparators(filePath), property)) << std::endl;
25}
26
27static bool validatePackage(Package &p, const QString &filePath, Checks checks, LogLevel logLevel)
28{
29 bool validPackage = true;
30
31 if (p.qtParts.isEmpty())
32 p.qtParts << u"libs"_s;
33
34 if (p.name.isEmpty()) {
35 if (p.id.startsWith(s: "chromium-"_L1)) // Ignore invalid README.chromium files
36 return false;
37
38 if (logLevel != SilentLog)
39 missingPropertyWarning(filePath, property: u"Name"_s);
40 validPackage = false;
41 }
42
43 if (p.id.isEmpty()) {
44 if (logLevel != SilentLog)
45 missingPropertyWarning(filePath, property: u"Id"_s);
46 validPackage = false;
47 }
48 if (p.license.isEmpty()) {
49 if (logLevel != SilentLog)
50 missingPropertyWarning(filePath, property: u"License"_s);
51 validPackage = false;
52 }
53
54 if (!p.copyright.isEmpty() && !p.copyrightFile.isEmpty()) {
55 if (logLevel != SilentLog) {
56 std::cerr << qPrintable(tr("File %1: Properties 'Copyright' and 'CopyrightFile' are "
57 "mutually exclusive.")
58 .arg(QDir::toNativeSeparators(filePath)))
59 << std::endl;
60 }
61 validPackage = false;
62 }
63
64 if (p.securityCritical && p.downloadLocation.isEmpty()) {
65 if (logLevel != SilentLog)
66 missingPropertyWarning(filePath, property: u"DownloadLocation"_s);
67 validPackage = false;
68 }
69
70 for (const QString &part : std::as_const(t&: p.qtParts)) {
71 if (part != "examples"_L1 && part != "tests"_L1
72 && part != "tools"_L1 && part != "libs"_L1) {
73
74 if (logLevel != SilentLog) {
75 std::cerr << qPrintable(tr("File %1: Property 'QtPart' contains unknown element "
76 "'%2'. Valid entries are 'examples', 'tests', 'tools' "
77 "and 'libs'.").arg(
78 QDir::toNativeSeparators(filePath), part))
79 << std::endl;
80 }
81 validPackage = false;
82 }
83 }
84
85 if (!(checks & Check::Paths))
86 return validPackage;
87
88 const QDir dir = p.path;
89 if (!dir.exists()) {
90 std::cerr << qPrintable(
91 tr("File %1: Directory '%2' does not exist.")
92 .arg(QDir::toNativeSeparators(filePath), QDir::toNativeSeparators(p.path)))
93 << std::endl;
94 validPackage = false;
95 } else {
96 for (const QString &file : std::as_const(t&: p.files)) {
97 if (!dir.exists(name: file)) {
98 if (logLevel != SilentLog) {
99 std::cerr << qPrintable(
100 tr("File %1: Path '%2' does not exist in directory '%3'.")
101 .arg(QDir::toNativeSeparators(filePath),
102 QDir::toNativeSeparators(file),
103 QDir::toNativeSeparators(p.path)))
104 << std::endl;
105 }
106 validPackage = false;
107 }
108 }
109 }
110
111 return validPackage;
112}
113
114static std::optional<QStringList> toStringList(const QJsonValue &value)
115{
116 if (!value.isArray())
117 return std::nullopt;
118 QStringList result;
119 for (const auto &iter : value.toArray()) {
120 if (iter.type() != QJsonValue::String)
121 return std::nullopt;
122 result.push_back(t: iter.toString());
123 }
124 return result;
125}
126
127static std::optional<QString> arrayToMultiLineString(const QJsonValue &value)
128{
129 if (!value.isArray())
130 return std::nullopt;
131 QString result;
132 for (const auto &iter : value.toArray()) {
133 if (iter.type() != QJsonValue::String)
134 return std::nullopt;
135 result.append(s: iter.toString());
136 result.append(s: QLatin1StringView("\n"));
137 }
138 return result;
139}
140
141// Extracts SPDX license ids from a SPDX license expression.
142// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
143static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
144{
145 const QStringList spdxOperators = {
146 u"AND"_s,
147 u"OR"_s,
148 u"WITH"_s
149 };
150
151 // Replace parentheses with spaces. We're not interested in grouping.
152 const QRegularExpression parensRegex(u"[()]"_s);
153 expression.replace(re: parensRegex, after: u" "_s);
154
155 // Split the string at space boundaries to extract tokens.
156 QStringList result;
157 for (const QString &token : expression.split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts)) {
158 if (spdxOperators.contains(str: token))
159 continue;
160
161 // Remove the unary + operator, if present.
162 if (token.endsWith(c: QLatin1Char('+')))
163 result.append(t: token.mid(position: 0, n: token.size() - 1));
164 else
165 result.append(t: token);
166 }
167 return result;
168}
169
170// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
171// Return a default-constructed QString if the directory was not found.
172static QString locateLicensesDir(const QString &packageDir)
173{
174 static const QString licensesSubDir = u"LICENSES"_s;
175 QDir dir(packageDir);
176 while (true) {
177 if (!dir.exists())
178 break;
179 if (dir.cd(dirName: licensesSubDir))
180 return dir.path();
181 if (dir.isRoot() || !dir.cdUp())
182 break;
183 }
184 return {};
185}
186
187// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
188// the specified package object.
189static bool autoDetectLicenseFiles(Package &p)
190{
191 const QString licensesDirPath = locateLicensesDir(packageDir: p.path);
192 const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(expression: p.licenseId);
193 if (!licenseIds.isEmpty() && licensesDirPath.isEmpty()) {
194 std::cerr << qPrintable(tr("LICENSES directory could not be located.")) << std::endl;
195 return false;
196 }
197
198 bool success = true;
199 QDir licensesDir(licensesDirPath);
200 for (const QString &id : licenseIds) {
201 QString fileName = id + u".txt";
202 if (licensesDir.exists(name: fileName)) {
203 p.licenseFiles.append(t: licensesDir.filePath(fileName));
204 } else {
205 std::cerr << qPrintable(tr("Expected license file not found: %1").arg(
206 QDir::toNativeSeparators(licensesDir.filePath(fileName))))
207 << std::endl;
208 success = false;
209 }
210 }
211
212 return success;
213}
214
215// Transforms a JSON object into a Package object
216static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
217 Checks checks, LogLevel logLevel)
218{
219 Package p;
220 bool validPackage = true;
221 const QString directory = QFileInfo(filePath).absolutePath();
222 p.path = directory;
223
224 for (auto iter = object.constBegin(); iter != object.constEnd(); ++iter) {
225 const QString key = iter.key();
226
227 if (!iter.value().isString() && key != "QtParts"_L1 && key != "SecurityCritical"_L1
228 && key != "Files"_L1 && key != "LicenseFiles"_L1 && key != "Comment"_L1
229 && key != "Copyright"_L1) {
230 if (logLevel != SilentLog)
231 std::cerr << qPrintable(tr("File %1: Expected JSON string as value of %2.").arg(
232 QDir::toNativeSeparators(filePath), key)) << std::endl;
233 validPackage = false;
234 continue;
235 }
236 const QString value = iter.value().toString();
237 if (key == "Name"_L1) {
238 p.name = value;
239 } else if (key == "Path"_L1) {
240 p.path = QDir(directory).absoluteFilePath(fileName: value);
241 } else if (key == "Files"_L1) {
242 QJsonValueConstRef jsonValue = iter.value();
243 if (jsonValue.isArray()) {
244 auto maybeStringList = toStringList(value: jsonValue);
245 if (maybeStringList)
246 p.files = maybeStringList.value();
247 } else if (jsonValue.isString()) {
248 // Legacy format: multiple values separated by space in one string.
249 p.files = value.simplified().split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts);
250 } else {
251 if (logLevel != SilentLog) {
252 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings as value "
253 "of Files."));
254 validPackage = false;
255 continue;
256 }
257 }
258 } else if (key == "Comment"_L1) {
259 // Accepted purely to record details of potential interest doing
260 // updates in future. Value is an arbitrary object. Any number of
261 // Comment entries may be present: JSON doesn't require names to be
262 // unique, albeit some linters may kvetch.
263 } else if (key == "Id"_L1) {
264 p.id = value;
265 } else if (key == "Homepage"_L1) {
266 p.homepage = value;
267 } else if (key == "Version"_L1) {
268 p.version = value;
269 } else if (key == "DownloadLocation"_L1) {
270 p.downloadLocation = value;
271 } else if (key == "License"_L1) {
272 p.license = value;
273 } else if (key == "LicenseId"_L1) {
274 p.licenseId = value;
275 } else if (key == "LicenseFile"_L1) {
276 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: value));
277 } else if (key == "LicenseFiles"_L1) {
278 auto strings = toStringList(value: iter.value());
279 if (!strings) {
280 if (logLevel != SilentLog)
281 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
282 .arg(QDir::toNativeSeparators(filePath), key))
283 << std::endl;
284 validPackage = false;
285 continue;
286 }
287 const QDir dir(directory);
288 for (const auto &iter : std::as_const(t&: strings.value()))
289 p.licenseFiles.push_back(t: dir.absoluteFilePath(fileName: iter));
290 } else if (key == "Copyright"_L1) {
291 QJsonValueConstRef jsonValue = iter.value();
292 if (jsonValue.isArray()) {
293 // Array joined with new lines
294 auto maybeString = arrayToMultiLineString(value: jsonValue);
295 if (maybeString)
296 p.copyright = maybeString.value();
297 } else if (jsonValue.isString()) {
298 // Legacy format: multiple values separated by space in one string.
299 p.copyright = value;
300 } else {
301 if (logLevel != SilentLog) {
302 std::cerr << qPrintable(tr("File %1: Expected JSON array of string or"
303 "string as value of %2.").arg(
304 QDir::toNativeSeparators(filePath), key)) << std::endl;
305 validPackage = false;
306 continue;
307 }
308 }
309 } else if (key == "CopyrightFile"_L1) {
310 p.copyrightFile = QDir(directory).absoluteFilePath(fileName: value);
311 } else if (key == "PackageComment"_L1) {
312 p.packageComment = value;
313 } else if (key == "QDocModule"_L1) {
314 p.qdocModule = value;
315 } else if (key == "Description"_L1) {
316 p.description = value;
317 } else if (key == "QtUsage"_L1) {
318 p.qtUsage = value;
319 } else if (key == "SecurityCritical"_L1) {
320 if (!iter.value().isBool()) {
321 std::cerr << qPrintable(tr("File %1: Expected JSON boolean in %2.")
322 .arg(QDir::toNativeSeparators(filePath), key))
323 << std::endl;
324 validPackage = false;
325 continue;
326 }
327 p.securityCritical = iter.value().toBool();
328 } else if (key == "QtParts"_L1) {
329 auto parts = toStringList(value: iter.value());
330 if (!parts) {
331 if (logLevel != SilentLog) {
332 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
333 .arg(QDir::toNativeSeparators(filePath), key))
334 << std::endl;
335 }
336 validPackage = false;
337 continue;
338 }
339 p.qtParts = parts.value();
340 } else {
341 if (logLevel != SilentLog) {
342 std::cerr << qPrintable(tr("File %1: Unknown key %2.").arg(
343 QDir::toNativeSeparators(filePath), key)) << std::endl;
344 }
345 validPackage = false;
346 }
347 }
348
349 if (!p.copyrightFile.isEmpty()) {
350 QFile file(p.copyrightFile);
351 if (!file.open(flags: QIODevice::ReadOnly)) {
352 std::cerr << qPrintable(tr("File %1: Cannot open 'CopyrightFile' %2.\n")
353 .arg(QDir::toNativeSeparators(filePath),
354 QDir::toNativeSeparators(p.copyrightFile)));
355 validPackage = false;
356 }
357 p.copyrightFileContents = QString::fromUtf8(ba: file.readAll());
358 }
359
360 foreach (const QString &licenseFile, p.licenseFiles) {
361 QFile file(licenseFile);
362 if (!file.open(flags: QIODevice::ReadOnly)) {
363 if (logLevel != SilentLog) {
364 std::cerr << qPrintable(tr("File %1: Cannot open 'LicenseFile' %2.\n")
365 .arg(QDir::toNativeSeparators(filePath),
366 QDir::toNativeSeparators(licenseFile)));
367 }
368 validPackage = false;
369 }
370 p.licenseFilesContents << QString::fromUtf8(ba: file.readAll()).trimmed();
371 }
372
373 if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
374 return std::nullopt;
375
376 if (!validatePackage(p, filePath, checks, logLevel) || !validPackage)
377 return std::nullopt;
378
379 return p;
380}
381
382// Parses a package's details from a README.chromium file
383static Package parseChromiumFile(QFile &file, const QString &filePath, LogLevel logLevel)
384{
385 const QString directory = QFileInfo(filePath).absolutePath();
386
387 // Parse the fields in the file
388 QHash<QString, QString> fields;
389
390 QTextStream in(&file);
391 while (!in.atEnd()) {
392 QString line = in.readLine().trimmed();
393 QStringList parts = line.split(sep: u":"_s);
394
395 if (parts.size() < 2)
396 continue;
397
398 QString key = parts.at(i: 0);
399 parts.removeFirst();
400 QString value = parts.join(sep: QString()).trimmed();
401
402 fields[key] = value;
403
404 if (line == "Description:"_L1) { // special field : should handle multi-lines values
405 while (!in.atEnd()) {
406 QString line = in.readLine().trimmed();
407
408 if (line.startsWith(s: "Local Modifications:"_L1)) // Don't include this part
409 break;
410
411 fields[key] += line + u"\n"_s;
412 }
413
414 break;
415 }
416 }
417
418 // Construct the Package object
419 Package p;
420
421 QString shortName = fields.contains(key: "Short Name"_L1)
422 ? fields["Short Name"_L1]
423 : fields["Name"_L1];
424 QString version = fields[u"Version"_s];
425
426 p.id = u"chromium-"_s + shortName.toLower().replace(c: QChar::Space, after: u"-"_s);
427 p.name = fields[u"Name"_s];
428 if (version != QLatin1Char('0')) // "0" : not applicable
429 p.version = version;
430 p.license = fields[u"License"_s];
431 p.homepage = fields[u"URL"_s];
432 p.qdocModule = u"qtwebengine"_s;
433 p.qtUsage = u"Used in Qt WebEngine"_s;
434 p.description = fields[u"Description"_s].trimmed();
435 p.path = directory;
436
437 QString licenseFile = fields[u"License File"_s];
438 if (licenseFile != QString() && licenseFile != "NOT_SHIPPED"_L1) {
439 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: licenseFile));
440 } else {
441 // Look for a LICENSE or COPYING file as a fallback
442 QDir dir = directory;
443
444 dir.setNameFilters({ u"LICENSE"_s, u"COPYING"_s });
445 dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
446
447 const QFileInfoList entries = dir.entryInfoList();
448 if (!entries.empty())
449 p.licenseFiles = QStringList(entries.at(i: 0).absoluteFilePath());
450 }
451
452 // let's ignore warnings regarding Chromium files for now
453 Q_UNUSED(validatePackage(p, filePath, {}, logLevel));
454
455 return p;
456}
457
458std::optional<QList<Package>> readFile(const QString &filePath, Checks checks, LogLevel logLevel)
459{
460 QList<Package> packages;
461 bool errorsFound = false;
462
463 if (logLevel == VerboseLog) {
464 std::cerr << qPrintable(tr("Reading file %1...").arg(
465 QDir::toNativeSeparators(filePath))) << std::endl;
466 }
467 QFile file(filePath);
468 if (!file.open(flags: QIODevice::ReadOnly | QIODevice::Text)) {
469 if (logLevel != SilentLog)
470 std::cerr << qPrintable(tr("Could not open file %1.").arg(
471 QDir::toNativeSeparators(file.fileName()))) << std::endl;
472 return std::nullopt;
473 }
474
475 if (filePath.endsWith(s: ".json"_L1)) {
476 QJsonParseError jsonParseError;
477 const QJsonDocument document = QJsonDocument::fromJson(json: file.readAll(), error: &jsonParseError);
478 if (document.isNull()) {
479 if (logLevel != SilentLog)
480 std::cerr << qPrintable(tr("Could not parse file %1: %2").arg(
481 QDir::toNativeSeparators(file.fileName()),
482 jsonParseError.errorString()))
483 << std::endl;
484 return std::nullopt;
485 }
486
487 if (document.isObject()) {
488 std::optional<Package> p =
489 readPackage(object: document.object(), filePath: file.fileName(), checks, logLevel);
490 if (p) {
491 packages << *p;
492 } else {
493 errorsFound = true;
494 }
495 } else if (document.isArray()) {
496 QJsonArray array = document.array();
497 for (int i = 0, size = array.size(); i < size; ++i) {
498 QJsonValue value = array.at(i);
499 if (value.isObject()) {
500 std::optional<Package> p =
501 readPackage(object: value.toObject(), filePath: file.fileName(), checks, logLevel);
502 if (p) {
503 packages << *p;
504 } else {
505 errorsFound = true;
506 }
507 } else {
508 if (logLevel != SilentLog) {
509 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.")
510 .arg(QDir::toNativeSeparators(file.fileName())))
511 << std::endl;
512 }
513 errorsFound = true;
514 }
515 }
516 } else {
517 if (logLevel != SilentLog) {
518 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.").arg(
519 QDir::toNativeSeparators(file.fileName()))) << std::endl;
520 }
521 errorsFound = true;
522 }
523 } else if (filePath.endsWith(s: ".chromium"_L1)) {
524 Package chromiumPackage = parseChromiumFile(file, filePath, logLevel);
525 if (!chromiumPackage.name.isEmpty()) // Skip invalid README.chromium files
526 packages << chromiumPackage;
527 } else {
528 if (logLevel != SilentLog) {
529 std::cerr << qPrintable(tr("File %1: Unsupported file type.")
530 .arg(QDir::toNativeSeparators(file.fileName())))
531 << std::endl;
532 }
533 errorsFound = true;
534 }
535
536 if (errorsFound)
537 return std::nullopt;
538 return packages;
539}
540
541std::optional<QList<Package>> scanDirectory(const QString &directory, InputFormats inputFormats,
542 Checks checks, LogLevel logLevel)
543{
544 QDir dir(directory);
545 QList<Package> packages;
546 bool errorsFound = false;
547
548 QStringList nameFilters = QStringList();
549 if (inputFormats & InputFormat::QtAttributions)
550 nameFilters << u"qt_attribution.json"_s;
551 if (inputFormats & InputFormat::ChromiumAttributions)
552 nameFilters << u"README.chromium"_s;
553 if (qEnvironmentVariableIsSet(varName: "QT_ATTRIBUTIONSSCANNER_TEST"))
554 nameFilters << u"qt_attribution_test.json"_s << u"README_test.chromium"_s;
555
556 dir.setNameFilters(nameFilters);
557 dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Files);
558
559 const QFileInfoList entries = dir.entryInfoList();
560 for (const QFileInfo &info : entries) {
561 if (info.isDir()) {
562 std::optional<QList<Package>> ps =
563 scanDirectory(directory: info.filePath(), inputFormats, checks, logLevel);
564 if (!ps)
565 errorsFound = true;
566 else
567 packages += *ps;
568 } else {
569 std::optional p = readFile(filePath: info.filePath(), checks, logLevel);
570 if (!p)
571 errorsFound = true;
572 else
573 packages += *p;
574 }
575 }
576
577 if (errorsFound)
578 return std::nullopt;
579 return packages;
580}
581
582} // namespace Scanner
583

source code of qttools/src/qtattributionsscanner/scanner.cpp