1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "assetdownloader.h"
5
6#include "tasking/concurrentcall.h"
7#include "tasking/networkquery.h"
8#include "tasking/tasktreerunner.h"
9
10#include <QtCore/private/qzipreader_p.h>
11
12#include <QtCore/QDir>
13#include <QtCore/QFile>
14#include <QtCore/QJsonArray>
15#include <QtCore/QJsonDocument>
16#include <QtCore/QJsonObject>
17#include <QtCore/QStandardPaths>
18#include <QtCore/QTemporaryDir>
19#include <QtCore/QTemporaryFile>
20
21using namespace Tasking;
22
23QT_BEGIN_NAMESPACE
24
25namespace Assets::Downloader {
26
27struct DownloadableAssets
28{
29 QUrl remoteUrl;
30 QList<QUrl> files;
31};
32
33class AssetDownloaderPrivate
34{
35public:
36 AssetDownloaderPrivate(AssetDownloader *q) : m_q(q) {}
37 AssetDownloader *m_q = nullptr;
38
39 std::unique_ptr<QNetworkAccessManager> m_manager;
40 std::unique_ptr<QTemporaryDir> m_temporaryDir;
41 TaskTreeRunner m_taskTreeRunner;
42 QString m_lastProgressText;
43 QDir m_localDownloadDir;
44
45 QString m_jsonFileName;
46 QString m_zipFileName;
47 QDir m_preferredLocalDownloadDir =
48 QStandardPaths::writableLocation(type: QStandardPaths::AppLocalDataLocation);
49 QUrl m_offlineAssetsFilePath;
50 QUrl m_downloadBase;
51
52 void setLocalDownloadDir(const QDir &dir)
53 {
54 if (m_localDownloadDir != dir) {
55 m_localDownloadDir = dir;
56 emit m_q->localDownloadDirChanged(url: QUrl::fromLocalFile(localfile: m_localDownloadDir.absolutePath()));
57 }
58 }
59 void setProgress(int progressValue, int progressMaximum, const QString &progressText)
60 {
61 m_lastProgressText = progressText;
62 emit m_q->progressChanged(progressValue, progressMaximum, progressText);
63 }
64 void updateProgress(int progressValue, int progressMaximum)
65 {
66 setProgress(progressValue, progressMaximum, progressText: m_lastProgressText);
67 }
68 void clearProgress(const QString &progressText)
69 {
70 setProgress(progressValue: 0, progressMaximum: 0, progressText);
71 }
72
73 void setupDownload(NetworkQuery *query, const QString &progressText)
74 {
75 query->setNetworkAccessManager(m_manager.get());
76 clearProgress(progressText);
77 QObject::connect(sender: query, signal: &NetworkQuery::started, context: query, slot: [this, query] {
78 QNetworkReply *reply = query->reply();
79 QObject::connect(sender: reply, signal: &QNetworkReply::downloadProgress,
80 context: query, slot: [this](qint64 bytesReceived, qint64 totalBytes) {
81 updateProgress(progressValue: (totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0, progressMaximum: 100);
82 });
83 });
84 }
85};
86
87static bool isWritableDir(const QDir &dir)
88{
89 if (dir.exists()) {
90 QTemporaryFile file(dir.filePath(fileName: QString::fromLatin1(ba: "tmp")));
91 return file.open();
92 }
93 return false;
94}
95
96static bool sameFileContent(const QFileInfo &first, const QFileInfo &second)
97{
98 if (first.exists() ^ second.exists())
99 return false;
100
101 if (first.size() != second.size())
102 return false;
103
104 QFile firstFile(first.absoluteFilePath());
105 QFile secondFile(second.absoluteFilePath());
106
107 if (firstFile.open(flags: QFile::ReadOnly) && secondFile.open(flags: QFile::ReadOnly)) {
108 char char1;
109 char char2;
110 int readBytes1 = 0;
111 int readBytes2 = 0;
112 while (!firstFile.atEnd()) {
113 readBytes1 = firstFile.read(data: &char1, maxlen: 1);
114 readBytes2 = secondFile.read(data: &char2, maxlen: 1);
115 if (readBytes1 != readBytes2 || readBytes1 != 1)
116 return false;
117 if (char1 != char2)
118 return false;
119 }
120 return true;
121 }
122
123 return false;
124}
125
126static bool createDirectory(const QDir &dir)
127{
128 if (dir.exists())
129 return true;
130
131 if (!createDirectory(dir: dir.absoluteFilePath(fileName: QString::fromUtf8(utf8: ".."))))
132 return false;
133
134 return dir.mkpath(dirPath: QString::fromUtf8(utf8: "."));
135}
136
137static bool canBeALocalBaseDir(const QDir &dir)
138{
139 if (dir.exists())
140 return !dir.isEmpty() || isWritableDir(dir);
141 return createDirectory(dir) && isWritableDir(dir);
142}
143
144static QDir baseLocalDir(const QDir &preferredLocalDir)
145{
146 if (canBeALocalBaseDir(dir: preferredLocalDir))
147 return preferredLocalDir;
148
149 qWarning().noquote() << "AssetDownloader: Cannot set \"" << preferredLocalDir
150 << "\" as a local download directory!";
151 return QStandardPaths::writableLocation(type: QStandardPaths::AppLocalDataLocation);
152}
153
154static QString pathFromUrl(const QUrl &url)
155{
156 if (url.isLocalFile())
157 return url.toLocalFile();
158
159 if (url.scheme() == u"qrc")
160 return u":" + url.path();
161
162 return url.toString();
163}
164
165static QList<QUrl> filterDownloadableAssets(const QList<QUrl> &assetFiles, const QDir &expectedDir)
166{
167 QList<QUrl> downloadList;
168 std::copy_if(first: assetFiles.begin(), last: assetFiles.end(), result: std::back_inserter(x&: downloadList),
169 pred: [&](const QUrl &assetPath) {
170 return !QFileInfo::exists(file: expectedDir.absoluteFilePath(fileName: assetPath.toString()));
171 });
172 return downloadList;
173}
174
175static bool allAssetsPresent(const QList<QUrl> &assetFiles, const QDir &expectedDir)
176{
177 return std::all_of(first: assetFiles.begin(), last: assetFiles.end(), pred: [&](const QUrl &assetPath) {
178 return QFileInfo::exists(file: expectedDir.absoluteFilePath(fileName: assetPath.toString()));
179 });
180}
181
182AssetDownloader::AssetDownloader(QObject *parent)
183 : QObject(parent)
184 , d(new AssetDownloaderPrivate(this))
185{}
186
187AssetDownloader::~AssetDownloader() = default;
188
189QUrl AssetDownloader::downloadBase() const
190{
191 return d->m_downloadBase;
192}
193
194void AssetDownloader::setDownloadBase(const QUrl &downloadBase)
195{
196 if (d->m_downloadBase != downloadBase) {
197 d->m_downloadBase = downloadBase;
198 emit downloadBaseChanged(d->m_downloadBase);
199 }
200}
201
202QUrl AssetDownloader::preferredLocalDownloadDir() const
203{
204 return QUrl::fromLocalFile(localfile: d->m_preferredLocalDownloadDir.absolutePath());
205}
206
207void AssetDownloader::setPreferredLocalDownloadDir(const QUrl &localDir)
208{
209 if (localDir.scheme() == u"qrc") {
210 qWarning() << "Cannot set a qrc as preferredLocalDownloadDir";
211 return;
212 }
213
214 const QString path = pathFromUrl(url: localDir);
215 if (d->m_preferredLocalDownloadDir != path) {
216 d->m_preferredLocalDownloadDir.setPath(path);
217 emit preferredLocalDownloadDirChanged(url: preferredLocalDownloadDir());
218 }
219}
220
221QUrl AssetDownloader::offlineAssetsFilePath() const
222{
223 return d->m_offlineAssetsFilePath;
224}
225
226void AssetDownloader::setOfflineAssetsFilePath(const QUrl &offlineAssetsFilePath)
227{
228 if (d->m_offlineAssetsFilePath != offlineAssetsFilePath) {
229 d->m_offlineAssetsFilePath = offlineAssetsFilePath;
230 emit offlineAssetsFilePathChanged(d->m_offlineAssetsFilePath);
231 }
232}
233
234QString AssetDownloader::jsonFileName() const
235{
236 return d->m_jsonFileName;
237}
238
239void AssetDownloader::setJsonFileName(const QString &jsonFileName)
240{
241 if (d->m_jsonFileName != jsonFileName) {
242 d->m_jsonFileName = jsonFileName;
243 emit jsonFileNameChanged(d->m_jsonFileName);
244 }
245}
246
247QString AssetDownloader::zipFileName() const
248{
249 return d->m_zipFileName;
250}
251
252void AssetDownloader::setZipFileName(const QString &zipFileName)
253{
254 if (d->m_zipFileName != zipFileName) {
255 d->m_zipFileName = zipFileName;
256 emit zipFileNameChanged(d->m_zipFileName);
257 }
258}
259
260QUrl AssetDownloader::localDownloadDir() const
261{
262 return QUrl::fromLocalFile(localfile: d->m_localDownloadDir.absolutePath());
263}
264
265static void precheckLocalFile(const QUrl &url)
266{
267 if (url.isEmpty())
268 return;
269 QFile file(pathFromUrl(url));
270 if (!file.open(flags: QIODevice::ReadOnly))
271 qWarning() << "Cannot open local file" << url;
272}
273
274static void readAssetsFileContent(QPromise<DownloadableAssets> &promise, const QByteArray &content)
275{
276 const QJsonObject json = QJsonDocument::fromJson(json: content).object();
277 const QJsonArray assetsArray = json[u"assets"].toArray();
278 DownloadableAssets result;
279 result.remoteUrl = json[u"url"].toString();
280 for (const QJsonValue &asset : assetsArray) {
281 if (promise.isCanceled())
282 return;
283 result.files.append(t: asset.toString());
284 }
285
286 if (result.files.isEmpty() || result.remoteUrl.isEmpty())
287 promise.future().cancel();
288 else
289 promise.addResult(result);
290}
291
292static void unzip(QPromise<void> &promise, const QByteArray &content, const QDir &directory,
293 const QString &fileName)
294{
295 const QString zipFilePath = directory.absoluteFilePath(fileName);
296 QFile zipFile(zipFilePath);
297 if (!zipFile.open(flags: QIODevice::WriteOnly)) {
298 promise.future().cancel();
299 return;
300 }
301 zipFile.write(data: content);
302 zipFile.close();
303
304 if (promise.isCanceled())
305 return;
306
307 QZipReader reader(zipFilePath);
308 const bool extracted = reader.extractAll(destinationDir: directory.absolutePath());
309 reader.close();
310 if (extracted)
311 QFile::remove(fileName: zipFilePath);
312 else
313 promise.future().cancel();
314}
315
316static void writeAsset(QPromise<void> &promise, const QByteArray &content, const QString &filePath)
317{
318 const QFileInfo fileInfo(filePath);
319 QFile file(fileInfo.absoluteFilePath());
320 if (!createDirectory(dir: fileInfo.dir()) || !file.open(flags: QFile::WriteOnly)) {
321 promise.future().cancel();
322 return;
323 }
324
325 if (promise.isCanceled())
326 return;
327
328 file.write(data: content);
329 file.close();
330}
331
332static void copyAndCheck(QPromise<void> &promise, const QString &sourcePath, const QString &destPath)
333{
334 QFile sourceFile(sourcePath);
335 QFile destFile(destPath);
336 const QFileInfo sourceFileInfo(sourceFile.fileName());
337 const QFileInfo destFileInfo(destFile.fileName());
338
339 if (destFile.exists() && !destFile.remove()) {
340 qWarning().noquote() << QString::fromLatin1(ba: "Unable to remove file \"%1\".")
341 .arg(a: QFileInfo(destFile.fileName()).absoluteFilePath());
342 promise.future().cancel();
343 return;
344 }
345
346 if (!createDirectory(dir: destFileInfo.absolutePath())) {
347 qWarning().noquote() << QString::fromLatin1(ba: "Cannot create directory \"%1\".")
348 .arg(a: destFileInfo.absolutePath());
349 promise.future().cancel();
350 return;
351 }
352
353 if (promise.isCanceled())
354 return;
355
356 if (!sourceFile.copy(newName: destFile.fileName()) && !sameFileContent(first: sourceFileInfo, second: destFileInfo))
357 promise.future().cancel();
358}
359
360void AssetDownloader::start()
361{
362 if (d->m_taskTreeRunner.isRunning())
363 return;
364
365 struct StorageData
366 {
367 QDir tempDir;
368 QByteArray jsonContent;
369 DownloadableAssets assets;
370 QList<QUrl> assetsToDownload;
371 QByteArray zipContent;
372 int doneCount = 0;
373 };
374
375 const Storage<StorageData> storage;
376
377 const auto onSetup = [this, storage] {
378 if (!d->m_manager)
379 d->m_manager = std::make_unique<QNetworkAccessManager>();
380 if (!d->m_temporaryDir)
381 d->m_temporaryDir = std::make_unique<QTemporaryDir>();
382 if (!d->m_temporaryDir->isValid()) {
383 qWarning() << "Cannot create a temporary directory.";
384 return SetupResult::StopWithError;
385 }
386 storage->tempDir = d->m_temporaryDir->path();
387 d->setLocalDownloadDir(baseLocalDir(preferredLocalDir: d->m_preferredLocalDownloadDir));
388 precheckLocalFile(url: resolvedUrl(url: d->m_offlineAssetsFilePath));
389 return SetupResult::Continue;
390 };
391
392 const auto onJsonDownloadSetup = [this](NetworkQuery &query) {
393 query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(relative: d->m_jsonFileName)));
394 d->setupDownload(query: &query, progressText: tr(s: "Downloading JSON file..."));
395 };
396 const auto onJsonDownloadDone = [this, storage](const NetworkQuery &query, DoneWith result) {
397 if (result == DoneWith::Success) {
398 storage->jsonContent = query.reply()->readAll();
399 return DoneResult::Success;
400 }
401 qWarning() << "Cannot download" << d->m_downloadBase.resolved(relative: d->m_jsonFileName)
402 << query.reply()->errorString();
403 if (d->m_offlineAssetsFilePath.isEmpty()) {
404 qWarning() << "Also there is no local file as a replacement";
405 return DoneResult::Error;
406 }
407
408 QFile file(pathFromUrl(url: resolvedUrl(url: d->m_offlineAssetsFilePath)));
409 if (!file.open(flags: QIODevice::ReadOnly)) {
410 qWarning() << "Also failed to open" << d->m_offlineAssetsFilePath;
411 return DoneResult::Error;
412 }
413
414 storage->jsonContent = file.readAll();
415 return DoneResult::Success;
416 };
417
418 const auto onReadAssetsFileSetup = [storage](ConcurrentCall<DownloadableAssets> &async) {
419 async.setConcurrentCallData(function&: readAssetsFileContent, args&: storage->jsonContent);
420 };
421 const auto onReadAssetsFileDone = [storage](const ConcurrentCall<DownloadableAssets> &async) {
422 storage->assets = async.result();
423 storage->assetsToDownload = storage->assets.files;
424 };
425
426 const auto onSkipIfAllAssetsPresent = [this, storage] {
427 return allAssetsPresent(assetFiles: storage->assets.files, expectedDir: d->m_localDownloadDir)
428 ? SetupResult::StopWithSuccess : SetupResult::Continue;
429 };
430
431 const auto onZipDownloadSetup = [this, storage](NetworkQuery &query) {
432 if (d->m_zipFileName.isEmpty())
433 return SetupResult::StopWithSuccess;
434
435 query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(relative: d->m_zipFileName)));
436 d->setupDownload(query: &query, progressText: tr(s: "Downloading zip file..."));
437 return SetupResult::Continue;
438 };
439 const auto onZipDownloadDone = [storage](const NetworkQuery &query, DoneWith result) {
440 if (result == DoneWith::Success)
441 storage->zipContent = query.reply()->readAll();
442 return DoneResult::Success; // Ignore zip download failure
443 };
444
445 const auto onUnzipSetup = [this, storage](ConcurrentCall<void> &async) {
446 if (storage->zipContent.isEmpty())
447 return SetupResult::StopWithSuccess;
448
449 async.setConcurrentCallData(function&: unzip, args&: storage->zipContent, args&: storage->tempDir, args&: d->m_zipFileName);
450 d->clearProgress(progressText: tr(s: "Unzipping..."));
451 return SetupResult::Continue;
452 };
453 const auto onUnzipDone = [storage](DoneWith result) {
454 if (result == DoneWith::Success) {
455 // Avoid downloading assets that are present in unzipped tree
456 StorageData &storageData = *storage;
457 storageData.assetsToDownload =
458 filterDownloadableAssets(assetFiles: storageData.assets.files, expectedDir: storageData.tempDir);
459 } else {
460 qWarning() << "ZipFile failed";
461 }
462 return DoneResult::Success; // Ignore unzip failure
463 };
464
465 const LoopUntil downloadIterator([storage](int iteration) {
466 return iteration < storage->assetsToDownload.count();
467 });
468
469 const Storage<QByteArray> assetStorage;
470
471 const auto onAssetsDownloadGroupSetup = [this, storage] {
472 d->setProgress(progressValue: 0, progressMaximum: storage->assetsToDownload.size(), progressText: tr(s: "Downloading assets..."));
473 };
474
475 const auto onAssetDownloadSetup = [this, storage, downloadIterator](NetworkQuery &query) {
476 query.setNetworkAccessManager(d->m_manager.get());
477 query.setRequest(QNetworkRequest(storage->assets.remoteUrl.resolved(
478 relative: storage->assetsToDownload.at(i: downloadIterator.iteration()))));
479 };
480 const auto onAssetDownloadDone = [assetStorage](const NetworkQuery &query, DoneWith result) {
481 if (result == DoneWith::Success)
482 *assetStorage = query.reply()->readAll();
483 };
484
485 const auto onAssetWriteSetup = [storage, downloadIterator, assetStorage](
486 ConcurrentCall<void> &async) {
487 const QString filePath = storage->tempDir.absoluteFilePath(
488 fileName: storage->assetsToDownload.at(i: downloadIterator.iteration()).toString());
489 async.setConcurrentCallData(function&: writeAsset, args&: *assetStorage, args: filePath);
490 };
491 const auto onAssetWriteDone = [this, storage](DoneWith result) {
492 if (result != DoneWith::Success) {
493 qWarning() << "Asset write failed";
494 return;
495 }
496 StorageData &storageData = *storage;
497 ++storageData.doneCount;
498 d->updateProgress(progressValue: storageData.doneCount, progressMaximum: storageData.assetsToDownload.size());
499 };
500
501 const LoopUntil copyIterator([storage](int iteration) {
502 return iteration < storage->assets.files.count();
503 });
504
505 const auto onAssetsCopyGroupSetup = [this, storage] {
506 storage->doneCount = 0;
507 d->setProgress(progressValue: 0, progressMaximum: storage->assets.files.size(), progressText: tr(s: "Copying assets..."));
508 };
509
510 const auto onAssetCopySetup = [this, storage, copyIterator](ConcurrentCall<void> &async) {
511 const QString fileName = storage->assets.files.at(i: copyIterator.iteration()).toString();
512 const QString sourcePath = storage->tempDir.absoluteFilePath(fileName);
513 const QString destPath = d->m_localDownloadDir.absoluteFilePath(fileName);
514 async.setConcurrentCallData(function&: copyAndCheck, args: sourcePath, args: destPath);
515 };
516 const auto onAssetCopyDone = [this, storage] {
517 StorageData &storageData = *storage;
518 ++storageData.doneCount;
519 d->updateProgress(progressValue: storageData.doneCount, progressMaximum: storageData.assets.files.size());
520 };
521
522 const auto onAssetsCopyGroupDone = [this, storage](DoneWith result) {
523 if (result != DoneWith::Success) {
524 d->setLocalDownloadDir(storage->tempDir);
525 qWarning() << "Asset copy failed";
526 return;
527 }
528 d->m_temporaryDir.reset();
529 };
530
531 const Group recipe {
532 storage,
533 onGroupSetup(handler: onSetup),
534 NetworkQueryTask(onJsonDownloadSetup, onJsonDownloadDone),
535 ConcurrentCallTask<DownloadableAssets>(onReadAssetsFileSetup, onReadAssetsFileDone, CallDoneIf::Success),
536 Group {
537 onGroupSetup(handler: onSkipIfAllAssetsPresent),
538 NetworkQueryTask(onZipDownloadSetup, onZipDownloadDone),
539 ConcurrentCallTask<void>(onUnzipSetup, onUnzipDone),
540 For {
541 downloadIterator,
542 parallelIdealThreadCountLimit,
543 onGroupSetup(handler: onAssetsDownloadGroupSetup),
544 Group {
545 assetStorage,
546 NetworkQueryTask(onAssetDownloadSetup, onAssetDownloadDone),
547 ConcurrentCallTask<void>(onAssetWriteSetup, onAssetWriteDone)
548 }
549 },
550 For {
551 copyIterator,
552 parallelIdealThreadCountLimit,
553 onGroupSetup(handler: onAssetsCopyGroupSetup),
554 ConcurrentCallTask<void>(onAssetCopySetup, onAssetCopyDone, CallDoneIf::Success),
555 onGroupDone(handler: onAssetsCopyGroupDone)
556 }
557 }
558 };
559 d->m_taskTreeRunner.start(recipe, setupHandler: [this](TaskTree *) { emit started(); },
560 doneHandler: [this](DoneWith result) { emit finished(success: result == DoneWith::Success); });
561}
562
563QUrl AssetDownloader::resolvedUrl(const QUrl &url) const
564{
565 return url;
566}
567
568} // namespace Assets::Downloader
569
570QT_END_NAMESPACE
571

source code of qtbase/src/assets/downloader/assetdownloader.cpp