1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "assetdownloader.h"
5
6#include "tasking/concurrentcall.h"
7#include "tasking/networkquery.h"
8#include "tasking/tasktreerunner.h"
9
10#include <QtCore/private/qzipreader_p.h>
11
12#include <QtCore/QDir>
13#include <QtCore/QFile>
14#include <QtCore/QJsonArray>
15#include <QtCore/QJsonDocument>
16#include <QtCore/QJsonObject>
17#include <QtCore/QStandardPaths>
18#include <QtCore/QTemporaryDir>
19#include <QtCore/QTemporaryFile>
20
21using namespace Tasking;
22
23QT_BEGIN_NAMESPACE
24
25namespace Assets::Downloader {
26
27struct DownloadableAssets
28{
29 QUrl remoteUrl;
30 QList<QUrl> files;
31};
32
33class AssetDownloaderPrivate
34{
35public:
36 AssetDownloaderPrivate(AssetDownloader *q) : m_q(q) {}
37 AssetDownloader *m_q = nullptr;
38
39 std::unique_ptr<QNetworkAccessManager> m_manager;
40 std::unique_ptr<QTemporaryDir> m_temporaryDir;
41 TaskTreeRunner m_taskTreeRunner;
42 QString m_lastProgressText;
43 QDir m_localDownloadDir;
44
45 QString m_jsonFileName;
46 QString m_zipFileName;
47 QDir m_preferredLocalDownloadDir =
48 QStandardPaths::writableLocation(type: QStandardPaths::AppLocalDataLocation);
49 QUrl m_offlineAssetsFilePath;
50 QUrl m_downloadBase;
51 QStringList m_networkErrors;
52 QStringList m_sslErrors;
53
54 void setLocalDownloadDir(const QDir &dir)
55 {
56 if (m_localDownloadDir != dir) {
57 m_localDownloadDir = dir;
58 emit m_q->localDownloadDirChanged(url: QUrl::fromLocalFile(localfile: m_localDownloadDir.absolutePath()));
59 }
60 }
61 void setProgress(int progressValue, int progressMaximum, const QString &progressText)
62 {
63 m_lastProgressText = progressText;
64 emit m_q->progressChanged(progressValue, progressMaximum, progressText);
65 }
66 void updateProgress(int progressValue, int progressMaximum)
67 {
68 setProgress(progressValue, progressMaximum, progressText: m_lastProgressText);
69 }
70 void clearProgress(const QString &progressText)
71 {
72 setProgress(progressValue: 0, progressMaximum: 0, progressText);
73 }
74
75 void setupDownload(NetworkQuery *query, const QString &progressText)
76 {
77 query->setNetworkAccessManager(m_manager.get());
78 clearProgress(progressText);
79 QObject::connect(sender: query, signal: &NetworkQuery::started, context: query, slot: [this, query] {
80 QNetworkReply *reply = query->reply();
81 QObject::connect(sender: reply, signal: &QNetworkReply::downloadProgress,
82 context: query, slot: [this](qint64 bytesReceived, qint64 totalBytes) {
83 updateProgress(progressValue: (totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0, progressMaximum: 100);
84 });
85 QObject::connect(sender: reply, signal: &QNetworkReply::errorOccurred, context: query, slot: [this, reply] {
86 m_networkErrors << reply->errorString();
87 });
88#if QT_CONFIG(ssl)
89 QObject::connect(sender: reply, signal: &QNetworkReply::sslErrors,
90 context: query, slot: [this](const QList<QSslError> &sslErrors) {
91 for (const QSslError &sslError : sslErrors)
92 m_sslErrors << sslError.errorString();
93 });
94#endif
95 });
96 }
97};
98
99static bool isWritableDir(const QDir &dir)
100{
101 if (dir.exists()) {
102 QTemporaryFile file(dir.filePath(fileName: QString::fromLatin1(ba: "tmp")));
103 return file.open();
104 }
105 return false;
106}
107
108static bool sameFileContent(const QFileInfo &first, const QFileInfo &second)
109{
110 if (first.exists() ^ second.exists())
111 return false;
112
113 if (first.size() != second.size())
114 return false;
115
116 QFile firstFile(first.absoluteFilePath());
117 QFile secondFile(second.absoluteFilePath());
118
119 if (firstFile.open(flags: QFile::ReadOnly) && secondFile.open(flags: QFile::ReadOnly)) {
120 char char1;
121 char char2;
122 int readBytes1 = 0;
123 int readBytes2 = 0;
124 while (!firstFile.atEnd()) {
125 readBytes1 = firstFile.read(data: &char1, maxlen: 1);
126 readBytes2 = secondFile.read(data: &char2, maxlen: 1);
127 if (readBytes1 != readBytes2 || readBytes1 != 1)
128 return false;
129 if (char1 != char2)
130 return false;
131 }
132 return true;
133 }
134
135 return false;
136}
137
138static bool createDirectory(const QDir &dir)
139{
140 if (dir.exists())
141 return true;
142
143 if (!createDirectory(dir: dir.absoluteFilePath(fileName: QString::fromUtf8(utf8: ".."))))
144 return false;
145
146 return dir.mkpath(dirPath: QString::fromUtf8(utf8: "."));
147}
148
149static bool canBeALocalBaseDir(const QDir &dir)
150{
151 if (dir.exists())
152 return !dir.isEmpty() || isWritableDir(dir);
153 return createDirectory(dir) && isWritableDir(dir);
154}
155
156static QDir baseLocalDir(const QDir &preferredLocalDir)
157{
158 if (canBeALocalBaseDir(dir: preferredLocalDir))
159 return preferredLocalDir;
160
161 qWarning().noquote() << "AssetDownloader: Cannot set \"" << preferredLocalDir
162 << "\" as a local download directory!";
163 return QStandardPaths::writableLocation(type: QStandardPaths::AppLocalDataLocation);
164}
165
166static QString pathFromUrl(const QUrl &url)
167{
168 if (url.isLocalFile())
169 return url.toLocalFile();
170
171 if (url.scheme() == u"qrc")
172 return u":" + url.path();
173
174 return url.toString();
175}
176
177static QList<QUrl> filterDownloadableAssets(const QList<QUrl> &assetFiles, const QDir &expectedDir)
178{
179 QList<QUrl> downloadList;
180 std::copy_if(first: assetFiles.begin(), last: assetFiles.end(), result: std::back_inserter(x&: downloadList),
181 pred: [&](const QUrl &assetPath) {
182 return !QFileInfo::exists(file: expectedDir.absoluteFilePath(fileName: assetPath.toString()));
183 });
184 return downloadList;
185}
186
187static bool allAssetsPresent(const QList<QUrl> &assetFiles, const QDir &expectedDir)
188{
189 return std::all_of(first: assetFiles.begin(), last: assetFiles.end(), pred: [&](const QUrl &assetPath) {
190 return QFileInfo::exists(file: expectedDir.absoluteFilePath(fileName: assetPath.toString()));
191 });
192}
193
194AssetDownloader::AssetDownloader(QObject *parent)
195 : QObject(parent)
196 , d(new AssetDownloaderPrivate(this))
197{}
198
199AssetDownloader::~AssetDownloader() = default;
200
201QUrl AssetDownloader::downloadBase() const
202{
203 return d->m_downloadBase;
204}
205
206void AssetDownloader::setDownloadBase(const QUrl &downloadBase)
207{
208 if (d->m_downloadBase != downloadBase) {
209 d->m_downloadBase = downloadBase;
210 emit downloadBaseChanged(d->m_downloadBase);
211 }
212}
213
214QUrl AssetDownloader::preferredLocalDownloadDir() const
215{
216 return QUrl::fromLocalFile(localfile: d->m_preferredLocalDownloadDir.absolutePath());
217}
218
219void AssetDownloader::setPreferredLocalDownloadDir(const QUrl &localDir)
220{
221 if (localDir.scheme() == u"qrc") {
222 qWarning() << "Cannot set a qrc as preferredLocalDownloadDir";
223 return;
224 }
225
226 const QString path = pathFromUrl(url: localDir);
227 if (d->m_preferredLocalDownloadDir != path) {
228 d->m_preferredLocalDownloadDir.setPath(path);
229 emit preferredLocalDownloadDirChanged(url: preferredLocalDownloadDir());
230 }
231}
232
233QUrl AssetDownloader::offlineAssetsFilePath() const
234{
235 return d->m_offlineAssetsFilePath;
236}
237
238void AssetDownloader::setOfflineAssetsFilePath(const QUrl &offlineAssetsFilePath)
239{
240 if (d->m_offlineAssetsFilePath != offlineAssetsFilePath) {
241 d->m_offlineAssetsFilePath = offlineAssetsFilePath;
242 emit offlineAssetsFilePathChanged(d->m_offlineAssetsFilePath);
243 }
244}
245
246QString AssetDownloader::jsonFileName() const
247{
248 return d->m_jsonFileName;
249}
250
251void AssetDownloader::setJsonFileName(const QString &jsonFileName)
252{
253 if (d->m_jsonFileName != jsonFileName) {
254 d->m_jsonFileName = jsonFileName;
255 emit jsonFileNameChanged(d->m_jsonFileName);
256 }
257}
258
259QString AssetDownloader::zipFileName() const
260{
261 return d->m_zipFileName;
262}
263
264void AssetDownloader::setZipFileName(const QString &zipFileName)
265{
266 if (d->m_zipFileName != zipFileName) {
267 d->m_zipFileName = zipFileName;
268 emit zipFileNameChanged(d->m_zipFileName);
269 }
270}
271
272QUrl AssetDownloader::localDownloadDir() const
273{
274 return QUrl::fromLocalFile(localfile: d->m_localDownloadDir.absolutePath());
275}
276
277QStringList AssetDownloader::networkErrors() const
278{
279 return d->m_networkErrors;
280}
281
282QStringList AssetDownloader::sslErrors() const
283{
284 return d->m_sslErrors;
285}
286
287static void precheckLocalFile(const QUrl &url)
288{
289 if (url.isEmpty())
290 return;
291 QFile file(pathFromUrl(url));
292 if (!file.open(flags: QIODevice::ReadOnly))
293 qWarning() << "Cannot open local file" << url;
294}
295
296static void readAssetsFileContent(QPromise<DownloadableAssets> &promise, const QByteArray &content)
297{
298 const QJsonObject json = QJsonDocument::fromJson(json: content).object();
299 const QJsonArray assetsArray = json[u"assets"].toArray();
300 DownloadableAssets result;
301 result.remoteUrl = json[u"url"].toString();
302 for (const QJsonValue &asset : assetsArray) {
303 if (promise.isCanceled())
304 return;
305 result.files.append(t: asset.toString());
306 }
307
308 if (result.files.isEmpty() || result.remoteUrl.isEmpty())
309 promise.future().cancel();
310 else
311 promise.addResult(result);
312}
313
314static void unzip(QPromise<void> &promise, const QByteArray &content, const QDir &directory,
315 const QString &fileName)
316{
317 const QString zipFilePath = directory.absoluteFilePath(fileName);
318 QFile zipFile(zipFilePath);
319 if (!zipFile.open(flags: QIODevice::WriteOnly)) {
320 promise.future().cancel();
321 return;
322 }
323 zipFile.write(data: content);
324 zipFile.close();
325
326 if (promise.isCanceled())
327 return;
328
329 QZipReader reader(zipFilePath);
330 const bool extracted = reader.extractAll(destinationDir: directory.absolutePath());
331 reader.close();
332 if (extracted)
333 QFile::remove(fileName: zipFilePath);
334 else
335 promise.future().cancel();
336}
337
338static void writeAsset(QPromise<void> &promise, const QByteArray &content, const QString &filePath)
339{
340 const QFileInfo fileInfo(filePath);
341 QFile file(fileInfo.absoluteFilePath());
342 if (!createDirectory(dir: fileInfo.dir()) || !file.open(flags: QFile::WriteOnly)) {
343 promise.future().cancel();
344 return;
345 }
346
347 if (promise.isCanceled())
348 return;
349
350 file.write(data: content);
351 file.close();
352}
353
354static void copyAndCheck(QPromise<void> &promise, const QString &sourcePath, const QString &destPath)
355{
356 QFile sourceFile(sourcePath);
357 QFile destFile(destPath);
358 const QFileInfo sourceFileInfo(sourceFile.fileName());
359 const QFileInfo destFileInfo(destFile.fileName());
360
361 if (destFile.exists() && !destFile.remove()) {
362 qWarning().noquote() << QString::fromLatin1(ba: "Unable to remove file \"%1\".")
363 .arg(a: QFileInfo(destFile.fileName()).absoluteFilePath());
364 promise.future().cancel();
365 return;
366 }
367
368 if (!createDirectory(dir: destFileInfo.absolutePath())) {
369 qWarning().noquote() << QString::fromLatin1(ba: "Cannot create directory \"%1\".")
370 .arg(a: destFileInfo.absolutePath());
371 promise.future().cancel();
372 return;
373 }
374
375 if (promise.isCanceled())
376 return;
377
378 if (!sourceFile.copy(newName: destFile.fileName()) && !sameFileContent(first: sourceFileInfo, second: destFileInfo))
379 promise.future().cancel();
380}
381
382void AssetDownloader::start()
383{
384 if (d->m_taskTreeRunner.isRunning())
385 return;
386
387 struct StorageData
388 {
389 QDir tempDir;
390 QByteArray jsonContent;
391 DownloadableAssets assets;
392 QList<QUrl> assetsToDownload;
393 QByteArray zipContent;
394 int doneCount = 0;
395 };
396
397 const Storage<StorageData> storage;
398
399 const auto onSetup = [this, storage] {
400 if (!d->m_manager)
401 d->m_manager = std::make_unique<QNetworkAccessManager>();
402 if (!d->m_temporaryDir)
403 d->m_temporaryDir = std::make_unique<QTemporaryDir>();
404 if (!d->m_temporaryDir->isValid()) {
405 qWarning() << "Cannot create a temporary directory.";
406 return SetupResult::StopWithError;
407 }
408 storage->tempDir = d->m_temporaryDir->path();
409 d->setLocalDownloadDir(baseLocalDir(preferredLocalDir: d->m_preferredLocalDownloadDir));
410 d->m_networkErrors.clear();
411 d->m_sslErrors.clear();
412 precheckLocalFile(url: resolvedUrl(url: d->m_offlineAssetsFilePath));
413 return SetupResult::Continue;
414 };
415
416 const auto onJsonDownloadSetup = [this](NetworkQuery &query) {
417 query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(relative: d->m_jsonFileName)));
418 d->setupDownload(query: &query, progressText: tr(s: "Downloading JSON file..."));
419 };
420 const auto onJsonDownloadDone = [this, storage](const NetworkQuery &query, DoneWith result) {
421 if (result == DoneWith::Success) {
422 storage->jsonContent = query.reply()->readAll();
423 return DoneResult::Success;
424 }
425 qWarning() << "Cannot download" << d->m_downloadBase.resolved(relative: d->m_jsonFileName)
426 << query.reply()->errorString();
427 if (d->m_offlineAssetsFilePath.isEmpty()) {
428 qWarning() << "Also there is no local file as a replacement";
429 return DoneResult::Error;
430 }
431
432 QFile file(pathFromUrl(url: resolvedUrl(url: d->m_offlineAssetsFilePath)));
433 if (!file.open(flags: QIODevice::ReadOnly)) {
434 qWarning() << "Also failed to open" << d->m_offlineAssetsFilePath;
435 return DoneResult::Error;
436 }
437
438 storage->jsonContent = file.readAll();
439 return DoneResult::Success;
440 };
441
442 const auto onReadAssetsFileSetup = [storage](ConcurrentCall<DownloadableAssets> &async) {
443 async.setConcurrentCallData(function&: readAssetsFileContent, args&: storage->jsonContent);
444 };
445 const auto onReadAssetsFileDone = [storage](const ConcurrentCall<DownloadableAssets> &async) {
446 storage->assets = async.result();
447 storage->assetsToDownload = storage->assets.files;
448 };
449
450 const auto onSkipIfAllAssetsPresent = [this, storage] {
451 return allAssetsPresent(assetFiles: storage->assets.files, expectedDir: d->m_localDownloadDir)
452 ? SetupResult::StopWithSuccess : SetupResult::Continue;
453 };
454
455 const auto onZipDownloadSetup = [this, storage](NetworkQuery &query) {
456 if (d->m_zipFileName.isEmpty())
457 return SetupResult::StopWithSuccess;
458
459 query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(relative: d->m_zipFileName)));
460 d->setupDownload(query: &query, progressText: tr(s: "Downloading zip file..."));
461 return SetupResult::Continue;
462 };
463 const auto onZipDownloadDone = [storage](const NetworkQuery &query, DoneWith result) {
464 if (result == DoneWith::Success)
465 storage->zipContent = query.reply()->readAll();
466 return DoneResult::Success; // Ignore zip download failure
467 };
468
469 const auto onUnzipSetup = [this, storage](ConcurrentCall<void> &async) {
470 if (storage->zipContent.isEmpty())
471 return SetupResult::StopWithSuccess;
472
473 async.setConcurrentCallData(function&: unzip, args&: storage->zipContent, args&: storage->tempDir, args&: d->m_zipFileName);
474 d->clearProgress(progressText: tr(s: "Unzipping..."));
475 return SetupResult::Continue;
476 };
477 const auto onUnzipDone = [storage](DoneWith result) {
478 if (result == DoneWith::Success) {
479 // Avoid downloading assets that are present in unzipped tree
480 StorageData &storageData = *storage;
481 storageData.assetsToDownload =
482 filterDownloadableAssets(assetFiles: storageData.assets.files, expectedDir: storageData.tempDir);
483 } else {
484 qWarning() << "ZipFile failed";
485 }
486 return DoneResult::Success; // Ignore unzip failure
487 };
488
489 const LoopUntil downloadIterator([storage](int iteration) {
490 return iteration < storage->assetsToDownload.count();
491 });
492
493 const Storage<QByteArray> assetStorage;
494
495 const auto onAssetsDownloadGroupSetup = [this, storage] {
496 d->setProgress(progressValue: 0, progressMaximum: storage->assetsToDownload.size(), progressText: tr(s: "Downloading assets..."));
497 };
498
499 const auto onAssetDownloadSetup = [this, storage, downloadIterator](NetworkQuery &query) {
500 query.setNetworkAccessManager(d->m_manager.get());
501 query.setRequest(QNetworkRequest(storage->assets.remoteUrl.resolved(
502 relative: storage->assetsToDownload.at(i: downloadIterator.iteration()))));
503 };
504 const auto onAssetDownloadDone = [assetStorage](const NetworkQuery &query, DoneWith result) {
505 if (result == DoneWith::Success)
506 *assetStorage = query.reply()->readAll();
507 };
508
509 const auto onAssetWriteSetup = [storage, downloadIterator, assetStorage](
510 ConcurrentCall<void> &async) {
511 const QString filePath = storage->tempDir.absoluteFilePath(
512 fileName: storage->assetsToDownload.at(i: downloadIterator.iteration()).toString());
513 async.setConcurrentCallData(function&: writeAsset, args&: *assetStorage, args: filePath);
514 };
515 const auto onAssetWriteDone = [this, storage](DoneWith result) {
516 if (result != DoneWith::Success) {
517 qWarning() << "Asset write failed";
518 return;
519 }
520 StorageData &storageData = *storage;
521 ++storageData.doneCount;
522 d->updateProgress(progressValue: storageData.doneCount, progressMaximum: storageData.assetsToDownload.size());
523 };
524
525 const LoopUntil copyIterator([storage](int iteration) {
526 return iteration < storage->assets.files.count();
527 });
528
529 const auto onAssetsCopyGroupSetup = [this, storage] {
530 storage->doneCount = 0;
531 d->setProgress(progressValue: 0, progressMaximum: storage->assets.files.size(), progressText: tr(s: "Copying assets..."));
532 };
533
534 const auto onAssetCopySetup = [this, storage, copyIterator](ConcurrentCall<void> &async) {
535 const QString fileName = storage->assets.files.at(i: copyIterator.iteration()).toString();
536 const QString sourcePath = storage->tempDir.absoluteFilePath(fileName);
537 const QString destPath = d->m_localDownloadDir.absoluteFilePath(fileName);
538 async.setConcurrentCallData(function&: copyAndCheck, args: sourcePath, args: destPath);
539 };
540 const auto onAssetCopyDone = [this, storage] {
541 StorageData &storageData = *storage;
542 ++storageData.doneCount;
543 d->updateProgress(progressValue: storageData.doneCount, progressMaximum: storageData.assets.files.size());
544 };
545
546 const auto onAssetsCopyGroupDone = [this, storage](DoneWith result) {
547 if (result != DoneWith::Success) {
548 d->setLocalDownloadDir(storage->tempDir);
549 qWarning() << "Asset copy failed";
550 return;
551 }
552 d->m_temporaryDir.reset();
553 };
554
555 const Group recipe {
556 storage,
557 onGroupSetup(handler: onSetup),
558 NetworkQueryTask(onJsonDownloadSetup, onJsonDownloadDone),
559 ConcurrentCallTask<DownloadableAssets>(onReadAssetsFileSetup, onReadAssetsFileDone, CallDoneIf::Success),
560 Group {
561 onGroupSetup(handler: onSkipIfAllAssetsPresent),
562 NetworkQueryTask(onZipDownloadSetup, onZipDownloadDone),
563 ConcurrentCallTask<void>(onUnzipSetup, onUnzipDone),
564 For (downloadIterator) >> Do {
565 parallelIdealThreadCountLimit,
566 onGroupSetup(handler: onAssetsDownloadGroupSetup),
567 Group {
568 assetStorage,
569 NetworkQueryTask(onAssetDownloadSetup, onAssetDownloadDone),
570 ConcurrentCallTask<void>(onAssetWriteSetup, onAssetWriteDone)
571 }
572 },
573 For (copyIterator) >> Do {
574 parallelIdealThreadCountLimit,
575 onGroupSetup(handler: onAssetsCopyGroupSetup),
576 ConcurrentCallTask<void>(onAssetCopySetup, onAssetCopyDone, CallDoneIf::Success),
577 onGroupDone(handler: onAssetsCopyGroupDone)
578 }
579 }
580 };
581 d->m_taskTreeRunner.start(recipe, setupHandler: [this](TaskTree *) { emit started(); },
582 doneHandler: [this](DoneWith result) { emit finished(success: result == DoneWith::Success); });
583}
584
585QUrl AssetDownloader::resolvedUrl(const QUrl &url) const
586{
587 return url;
588}
589
590} // namespace Assets::Downloader
591
592QT_END_NAMESPACE
593

source code of qtbase/src/assets/downloader/assetdownloader.cpp