1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "scanner.h"
5#include "logging.h"
6
7#include <QtCore/qdir.h>
8#include <QtCore/qhash.h>
9#include <QtCore/qjsonarray.h>
10#include <QtCore/qjsondocument.h>
11#include <QtCore/qjsonobject.h>
12#include <QtCore/qtextstream.h>
13#include <QtCore/qvariant.h>
14
15#include <iostream>
16
17using namespace Qt::Literals::StringLiterals;
18
19namespace Scanner {
20
21static void missingPropertyWarning(const QString &filePath, const QString &property)
22{
23 std::cerr << qPrintable(tr("File %1: Missing mandatory property '%2'.").arg(
24 QDir::toNativeSeparators(filePath), property)) << std::endl;
25}
26
27static bool validatePackage(Package &p, const QString &filePath, Checks checks, LogLevel logLevel)
28{
29 bool validPackage = true;
30
31 if (p.qtParts.isEmpty())
32 p.qtParts << u"libs"_s;
33
34 if (p.name.isEmpty()) {
35 if (p.id.startsWith(s: "chromium-"_L1)) // Ignore invalid README.chromium files
36 return false;
37
38 if (logLevel != SilentLog)
39 missingPropertyWarning(filePath, property: u"Name"_s);
40 validPackage = false;
41 }
42
43 if (p.id.isEmpty()) {
44 if (logLevel != SilentLog)
45 missingPropertyWarning(filePath, property: u"Id"_s);
46 validPackage = false;
47 }
48 if (p.license.isEmpty()) {
49 if (logLevel != SilentLog)
50 missingPropertyWarning(filePath, property: u"License"_s);
51 validPackage = false;
52 }
53
54 if (!p.copyright.isEmpty() && !p.copyrightFile.isEmpty()) {
55 if (logLevel != SilentLog) {
56 std::cerr << qPrintable(tr("File %1: Properties 'Copyright' and 'CopyrightFile' are "
57 "mutually exclusive.")
58 .arg(QDir::toNativeSeparators(filePath)))
59 << std::endl;
60 }
61 validPackage = false;
62 }
63
64 if (p.securityCritical && p.downloadLocation.isEmpty()) {
65 if (logLevel != SilentLog)
66 missingPropertyWarning(filePath, property: u"DownloadLocation"_s);
67 validPackage = false;
68 }
69
70 for (const QString &part : std::as_const(t&: p.qtParts)) {
71 if (part != "examples"_L1 && part != "tests"_L1
72 && part != "tools"_L1 && part != "libs"_L1) {
73
74 if (logLevel != SilentLog) {
75 std::cerr << qPrintable(tr("File %1: Property 'QtPart' contains unknown element "
76 "'%2'. Valid entries are 'examples', 'tests', 'tools' "
77 "and 'libs'.").arg(
78 QDir::toNativeSeparators(filePath), part))
79 << std::endl;
80 }
81 validPackage = false;
82 }
83 }
84
85 if (!(checks & Check::Paths))
86 return validPackage;
87
88 const QDir dir = p.path;
89 if (!dir.exists()) {
90 std::cerr << qPrintable(
91 tr("File %1: Directory '%2' does not exist.")
92 .arg(QDir::toNativeSeparators(filePath), QDir::toNativeSeparators(p.path)))
93 << std::endl;
94 validPackage = false;
95 } else {
96 for (const QString &file : std::as_const(t&: p.files)) {
97 if (!dir.exists(name: file)) {
98 if (logLevel != SilentLog) {
99 std::cerr << qPrintable(
100 tr("File %1: Path '%2' does not exist in directory '%3'.")
101 .arg(QDir::toNativeSeparators(filePath),
102 QDir::toNativeSeparators(file),
103 QDir::toNativeSeparators(p.path)))
104 << std::endl;
105 }
106 validPackage = false;
107 }
108 }
109 }
110
111 return validPackage;
112}
113
114static std::optional<QStringList> toStringList(const QJsonValue &value)
115{
116 if (!value.isArray())
117 return std::nullopt;
118 QStringList result;
119 for (const auto &iter : value.toArray()) {
120 if (iter.type() != QJsonValue::String)
121 return std::nullopt;
122 result.push_back(t: iter.toString());
123 }
124 return result;
125}
126
127static std::optional<QString> arrayToMultiLineString(const QJsonValue &value)
128{
129 if (!value.isArray())
130 return std::nullopt;
131 QString result;
132 for (const auto &iter : value.toArray()) {
133 if (iter.type() != QJsonValue::String)
134 return std::nullopt;
135 result.append(s: iter.toString());
136 result.append(s: QLatin1StringView("\n"));
137 }
138 return result;
139}
140
141// Extracts SPDX license ids from a SPDX license expression.
142// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
143static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
144{
145 const QStringList spdxOperators = {
146 u"AND"_s,
147 u"OR"_s,
148 u"WITH"_s
149 };
150
151 // Replace parentheses with spaces. We're not interested in grouping.
152 const QRegularExpression parensRegex(u"[()]"_s);
153 expression.replace(re: parensRegex, after: u" "_s);
154
155 // Split the string at space boundaries to extract tokens.
156 QStringList result;
157 for (const QString &token : expression.split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts)) {
158 if (spdxOperators.contains(str: token))
159 continue;
160
161 // Remove the unary + operator, if present.
162 if (token.endsWith(c: QLatin1Char('+')))
163 result.append(t: token.mid(position: 0, n: token.size() - 1));
164 else
165 result.append(t: token);
166 }
167 return result;
168}
169
170// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
171// Return a default-constructed QString if the directory was not found.
172static QString locateLicensesDir(const QString &packageDir)
173{
174 static const QString licensesSubDir = u"LICENSES"_s;
175 QDir dir(packageDir);
176 while (true) {
177 if (!dir.exists())
178 break;
179 if (dir.cd(dirName: licensesSubDir))
180 return dir.path();
181 if (dir.isRoot() || !dir.cdUp())
182 break;
183 }
184 return {};
185}
186
187// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
188// the specified package object.
189static bool autoDetectLicenseFiles(Package &p)
190{
191 const QString licensesDirPath = locateLicensesDir(packageDir: p.path);
192 const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(expression: p.licenseId);
193 if (!licenseIds.isEmpty() && licensesDirPath.isEmpty()) {
194 std::cerr << qPrintable(tr("LICENSES directory could not be located.")) << std::endl;
195 return false;
196 }
197
198 bool success = true;
199 QDir licensesDir(licensesDirPath);
200 for (const QString &id : licenseIds) {
201 QString fileName = id + u".txt";
202 if (licensesDir.exists(name: fileName)) {
203 p.licenseFiles.append(t: licensesDir.filePath(fileName));
204 } else {
205 std::cerr << qPrintable(tr("Expected license file not found: %1").arg(
206 QDir::toNativeSeparators(licensesDir.filePath(fileName))))
207 << std::endl;
208 success = false;
209 }
210 }
211
212 return success;
213}
214
215// Tries to interpret a json value either as a string or an array of strings, and assigns the
216// result to outList. Returns true on success, false on failure. On failure, it also conditionally
217// prints an error.
218static bool handleStringOrStringArrayJsonKey(QStringList &outList, const QString &key,
219 QJsonValueConstRef jsonValue, const QString &filePath,
220 LogLevel logLevel)
221{
222 if (jsonValue.isArray()) {
223 auto maybeStringList = toStringList(value: jsonValue);
224 if (maybeStringList)
225 outList = maybeStringList.value();
226 } else if (jsonValue.isString()) {
227 outList.append(t: jsonValue.toString());
228 } else {
229 if (logLevel != SilentLog) {
230 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
231 "string as value of %2.").arg(
232 QDir::toNativeSeparators(filePath), key))
233 << std::endl;
234 }
235 return false;
236 }
237 return true;
238}
239
240// Transforms a JSON object into a Package object
241static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
242 Checks checks, LogLevel logLevel)
243{
244 Package p;
245 bool validPackage = true;
246 const QString directory = QFileInfo(filePath).absolutePath();
247 p.path = directory;
248
249 for (auto iter = object.constBegin(); iter != object.constEnd(); ++iter) {
250 const QString key = iter.key();
251
252 if (!iter.value().isString() && key != "QtParts"_L1 && key != "SecurityCritical"_L1
253 && key != "Files"_L1 && key != "LicenseFiles"_L1 && key != "Comment"_L1
254 && key != "Copyright"_L1 && key != "CPE"_L1 && key != "PURL"_L1) {
255 if (logLevel != SilentLog)
256 std::cerr << qPrintable(tr("File %1: Expected JSON string as value of %2.").arg(
257 QDir::toNativeSeparators(filePath), key)) << std::endl;
258 validPackage = false;
259 continue;
260 }
261 const QString value = iter.value().toString();
262 if (key == "Name"_L1) {
263 p.name = value;
264 } else if (key == "Path"_L1) {
265 p.path = QDir(directory).absoluteFilePath(fileName: value);
266 } else if (key == "Files"_L1) {
267 QJsonValueConstRef jsonValue = iter.value();
268 if (jsonValue.isArray()) {
269 auto maybeStringList = toStringList(value: jsonValue);
270 if (maybeStringList)
271 p.files = maybeStringList.value();
272 } else if (jsonValue.isString()) {
273 // Legacy format: multiple values separated by space in one string.
274 p.files = value.simplified().split(sep: QLatin1Char(' '), behavior: Qt::SkipEmptyParts);
275 } else {
276 if (logLevel != SilentLog) {
277 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings as value "
278 "of Files."));
279 validPackage = false;
280 continue;
281 }
282 }
283 } else if (key == "Comment"_L1) {
284 // Accepted purely to record details of potential interest doing
285 // updates in future. Value is an arbitrary object. Any number of
286 // Comment entries may be present: JSON doesn't require names to be
287 // unique, albeit some linters may kvetch.
288 } else if (key == "Id"_L1) {
289 p.id = value;
290 } else if (key == "Homepage"_L1) {
291 p.homepage = value;
292 } else if (key == "Version"_L1) {
293 p.version = value;
294 } else if (key == "DownloadLocation"_L1) {
295 p.downloadLocation = value;
296 } else if (key == "License"_L1) {
297 p.license = value;
298 } else if (key == "LicenseId"_L1) {
299 p.licenseId = value;
300 } else if (key == "LicenseFile"_L1) {
301 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: value));
302 } else if (key == "LicenseFiles"_L1) {
303 auto strings = toStringList(value: iter.value());
304 if (!strings) {
305 if (logLevel != SilentLog)
306 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
307 .arg(QDir::toNativeSeparators(filePath), key))
308 << std::endl;
309 validPackage = false;
310 continue;
311 }
312 const QDir dir(directory);
313 for (const auto &iter : std::as_const(t&: strings.value()))
314 p.licenseFiles.push_back(t: dir.absoluteFilePath(fileName: iter));
315 } else if (key == "Copyright"_L1) {
316 QJsonValueConstRef jsonValue = iter.value();
317 if (jsonValue.isArray()) {
318 // Array joined with new lines
319 auto maybeString = arrayToMultiLineString(value: jsonValue);
320 if (maybeString)
321 p.copyright = maybeString.value();
322 } else if (jsonValue.isString()) {
323 // Legacy format: multiple values separated by space in one string.
324 p.copyright = value;
325 } else {
326 if (logLevel != SilentLog) {
327 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
328 "string as value of %2.").arg(
329 QDir::toNativeSeparators(filePath), key)) << std::endl;
330 validPackage = false;
331 continue;
332 }
333 }
334 } else if (key == "CPE"_L1) {
335 const QJsonValueConstRef jsonValue = iter.value();
336 if (!handleStringOrStringArrayJsonKey(outList&: p.cpeList, key, jsonValue, filePath, logLevel)) {
337 validPackage = false;
338 continue;
339 }
340 } else if (key == "PURL"_L1) {
341 const QJsonValueConstRef jsonValue = iter.value();
342 if (!handleStringOrStringArrayJsonKey(outList&: p.purlList, key, jsonValue, filePath, logLevel)) {
343 validPackage = false;
344 continue;
345 }
346 } else if (key == "CopyrightFile"_L1) {
347 p.copyrightFile = QDir(directory).absoluteFilePath(fileName: value);
348 } else if (key == "PackageComment"_L1) {
349 p.packageComment = value;
350 } else if (key == "QDocModule"_L1) {
351 p.qdocModule = value;
352 } else if (key == "Description"_L1) {
353 p.description = value;
354 } else if (key == "QtUsage"_L1) {
355 p.qtUsage = value;
356 } else if (key == "SecurityCritical"_L1) {
357 if (!iter.value().isBool()) {
358 std::cerr << qPrintable(tr("File %1: Expected JSON boolean in %2.")
359 .arg(QDir::toNativeSeparators(filePath), key))
360 << std::endl;
361 validPackage = false;
362 continue;
363 }
364 p.securityCritical = iter.value().toBool();
365 } else if (key == "QtParts"_L1) {
366 auto parts = toStringList(value: iter.value());
367 if (!parts) {
368 if (logLevel != SilentLog) {
369 std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
370 .arg(QDir::toNativeSeparators(filePath), key))
371 << std::endl;
372 }
373 validPackage = false;
374 continue;
375 }
376 p.qtParts = parts.value();
377 } else {
378 if (logLevel != SilentLog) {
379 std::cerr << qPrintable(tr("File %1: Unknown key %2.").arg(
380 QDir::toNativeSeparators(filePath), key)) << std::endl;
381 }
382 validPackage = false;
383 }
384 }
385
386 if (!p.copyrightFile.isEmpty()) {
387 QFile file(p.copyrightFile);
388 if (!file.open(flags: QIODevice::ReadOnly)) {
389 std::cerr << qPrintable(tr("File %1: Cannot open 'CopyrightFile' %2.\n")
390 .arg(QDir::toNativeSeparators(filePath),
391 QDir::toNativeSeparators(p.copyrightFile)));
392 validPackage = false;
393 }
394 p.copyrightFileContents = QString::fromUtf8(ba: file.readAll());
395 }
396
397 if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
398 return std::nullopt;
399
400 for (const QString &licenseFile : std::as_const(t&: p.licenseFiles)) {
401 QFile file(licenseFile);
402 if (!file.open(flags: QIODevice::ReadOnly)) {
403 if (logLevel != SilentLog) {
404 std::cerr << qPrintable(tr("File %1: Cannot open 'LicenseFile' %2.\n")
405 .arg(QDir::toNativeSeparators(filePath),
406 QDir::toNativeSeparators(licenseFile)));
407 }
408 validPackage = false;
409 }
410 p.licenseFilesContents << QString::fromUtf8(ba: file.readAll()).trimmed();
411 }
412
413 if (!validatePackage(p, filePath, checks, logLevel) || !validPackage)
414 return std::nullopt;
415
416 return p;
417}
418
419// Parses a package's details from a README.chromium file
420static Package parseChromiumFile(QFile &file, const QString &filePath, LogLevel logLevel)
421{
422 const QString directory = QFileInfo(filePath).absolutePath();
423
424 // Parse the fields in the file
425 QHash<QString, QString> fields;
426
427 QTextStream in(&file);
428 while (!in.atEnd()) {
429 QString line = in.readLine().trimmed();
430 QStringList parts = line.split(sep: u":"_s);
431
432 if (parts.size() < 2)
433 continue;
434
435 QString key = parts.at(i: 0);
436 parts.removeFirst();
437 QString value = parts.join(sep: QString()).trimmed();
438
439 fields[key] = value;
440
441 if (line == "Description:"_L1) { // special field : should handle multi-lines values
442 while (!in.atEnd()) {
443 QString line = in.readLine().trimmed();
444
445 if (line.startsWith(s: "Local Modifications:"_L1)) // Don't include this part
446 break;
447
448 fields[key] += line + u"\n"_s;
449 }
450
451 break;
452 }
453 }
454
455 // Construct the Package object
456 Package p;
457
458 QString shortName = fields.contains(key: "Short Name"_L1)
459 ? fields["Short Name"_L1]
460 : fields["Name"_L1];
461 QString version = fields[u"Version"_s];
462
463 p.id = u"chromium-"_s + shortName.toLower().replace(c: QChar::Space, after: u"-"_s);
464 p.name = fields[u"Name"_s];
465 if (version != QLatin1Char('0')) // "0" : not applicable
466 p.version = version;
467 p.license = fields[u"License"_s];
468 p.homepage = fields[u"URL"_s];
469 p.qdocModule = u"qtwebengine"_s;
470 p.qtUsage = u"Used in Qt WebEngine"_s;
471 p.description = fields[u"Description"_s].trimmed();
472 p.path = directory;
473
474 QString licenseFile = fields[u"License File"_s];
475 if (licenseFile != QString() && licenseFile != "NOT_SHIPPED"_L1) {
476 p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(fileName: licenseFile));
477 } else {
478 // Look for a LICENSE or COPYING file as a fallback
479 QDir dir = directory;
480
481 dir.setNameFilters({ u"LICENSE"_s, u"COPYING"_s });
482 dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
483
484 const QFileInfoList entries = dir.entryInfoList();
485 if (!entries.empty())
486 p.licenseFiles = QStringList(entries.at(i: 0).absoluteFilePath());
487 }
488
489 // let's ignore warnings regarding Chromium files for now
490 Q_UNUSED(validatePackage(p, filePath, {}, logLevel));
491
492 return p;
493}
494
495std::optional<QList<Package>> readFile(const QString &filePath, Checks checks, LogLevel logLevel)
496{
497 QList<Package> packages;
498 bool errorsFound = false;
499
500 if (logLevel == VerboseLog) {
501 std::cerr << qPrintable(tr("Reading file %1...").arg(
502 QDir::toNativeSeparators(filePath))) << std::endl;
503 }
504 QFile file(filePath);
505 if (!file.open(flags: QIODevice::ReadOnly | QIODevice::Text)) {
506 if (logLevel != SilentLog)
507 std::cerr << qPrintable(tr("Could not open file %1.").arg(
508 QDir::toNativeSeparators(file.fileName()))) << std::endl;
509 return std::nullopt;
510 }
511
512 if (filePath.endsWith(s: ".json"_L1)) {
513 QJsonParseError jsonParseError;
514 const QJsonDocument document = QJsonDocument::fromJson(json: file.readAll(), error: &jsonParseError);
515 if (document.isNull()) {
516 if (logLevel != SilentLog)
517 std::cerr << qPrintable(tr("Could not parse file %1: %2").arg(
518 QDir::toNativeSeparators(file.fileName()),
519 jsonParseError.errorString()))
520 << std::endl;
521 return std::nullopt;
522 }
523
524 if (document.isObject()) {
525 std::optional<Package> p =
526 readPackage(object: document.object(), filePath: file.fileName(), checks, logLevel);
527 if (p) {
528 packages << *p;
529 } else {
530 errorsFound = true;
531 }
532 } else if (document.isArray()) {
533 QJsonArray array = document.array();
534 for (int i = 0, size = array.size(); i < size; ++i) {
535 QJsonValue value = array.at(i);
536 if (value.isObject()) {
537 std::optional<Package> p =
538 readPackage(object: value.toObject(), filePath: file.fileName(), checks, logLevel);
539 if (p) {
540 packages << *p;
541 } else {
542 errorsFound = true;
543 }
544 } else {
545 if (logLevel != SilentLog) {
546 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.")
547 .arg(QDir::toNativeSeparators(file.fileName())))
548 << std::endl;
549 }
550 errorsFound = true;
551 }
552 }
553 } else {
554 if (logLevel != SilentLog) {
555 std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.").arg(
556 QDir::toNativeSeparators(file.fileName()))) << std::endl;
557 }
558 errorsFound = true;
559 }
560 } else if (filePath.endsWith(s: ".chromium"_L1)) {
561 Package chromiumPackage = parseChromiumFile(file, filePath, logLevel);
562 if (!chromiumPackage.name.isEmpty()) // Skip invalid README.chromium files
563 packages << chromiumPackage;
564 } else {
565 if (logLevel != SilentLog) {
566 std::cerr << qPrintable(tr("File %1: Unsupported file type.")
567 .arg(QDir::toNativeSeparators(file.fileName())))
568 << std::endl;
569 }
570 errorsFound = true;
571 }
572
573 if (errorsFound)
574 return std::nullopt;
575 return packages;
576}
577
578std::optional<QList<Package>> scanDirectory(const QString &directory, InputFormats inputFormats,
579 Checks checks, LogLevel logLevel)
580{
581 QDir dir(directory);
582 QList<Package> packages;
583 bool errorsFound = false;
584
585 QStringList nameFilters = QStringList();
586 if (inputFormats & InputFormat::QtAttributions)
587 nameFilters << u"qt_attribution.json"_s;
588 if (inputFormats & InputFormat::ChromiumAttributions)
589 nameFilters << u"README.chromium"_s;
590 if (qEnvironmentVariableIsSet(varName: "QT_ATTRIBUTIONSSCANNER_TEST"))
591 nameFilters << u"qt_attribution_test.json"_s << u"README_test.chromium"_s;
592
593 dir.setNameFilters(nameFilters);
594 dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Files);
595
596 const QFileInfoList entries = dir.entryInfoList();
597 for (const QFileInfo &info : entries) {
598 if (info.isDir()) {
599 std::optional<QList<Package>> ps =
600 scanDirectory(directory: info.filePath(), inputFormats, checks, logLevel);
601 if (!ps)
602 errorsFound = true;
603 else
604 packages += *ps;
605 } else {
606 std::optional p = readFile(filePath: info.filePath(), checks, logLevel);
607 if (!p)
608 errorsFound = true;
609 else
610 packages += *p;
611 }
612 }
613
614 if (errorsFound)
615 return std::nullopt;
616 return packages;
617}
618
619} // namespace Scanner
620

Provided by KDAB

Privacy Policy
Learn to use CMake with our Intro Training
Find out more

source code of qttools/src/qtattributionsscanner/scanner.cpp