1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4//#define QNETWORKDISKCACHE_DEBUG
5
6
7#include "qnetworkdiskcache.h"
8#include "qnetworkdiskcache_p.h"
9#include "QtCore/qscopedpointer.h"
10
11#include <qfile.h>
12#include <qdir.h>
13#include <qdatastream.h>
14#include <qdatetime.h>
15#include <qdirlisting.h>
16#include <qurl.h>
17#include <qcryptographichash.h>
18#include <qdebug.h>
19
20#include <memory>
21
22#define CACHE_POSTFIX ".d"_L1
23#define CACHE_VERSION 8
24#define DATA_DIR "data"_L1
25
26#define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)
27
28QT_BEGIN_NAMESPACE
29
30using namespace Qt::StringLiterals;
31
32/*!
33 \class QNetworkDiskCache
34 \since 4.5
35 \inmodule QtNetwork
36
37 \brief The QNetworkDiskCache class provides a very basic disk cache.
38
39 QNetworkDiskCache stores each url in its own file inside of the
40 cacheDirectory using QDataStream. Files with a text MimeType
41 are compressed using qCompress. Data is written to disk only in insert()
42 and updateMetaData().
43
44 Currently you cannot share the same cache files with more than
45 one disk cache.
46
47 QNetworkDiskCache by default limits the amount of space that the cache will
48 use on the system to 50MB.
49
50 Note you have to set the cache directory before it will work.
51
52 A network disk cache can be enabled by:
53
54 \snippet code/src_network_access_qnetworkdiskcache.cpp 0
55
56 When sending requests, to control the preference of when to use the cache
57 and when to use the network, consider the following:
58
59 \snippet code/src_network_access_qnetworkdiskcache.cpp 1
60
61 To check whether the response came from the cache or from the network, the
62 following can be applied:
63
64 \snippet code/src_network_access_qnetworkdiskcache.cpp 2
65*/
66
67/*!
68 Creates a new disk cache. The \a parent argument is passed to
69 QAbstractNetworkCache's constructor.
70 */
71QNetworkDiskCache::QNetworkDiskCache(QObject *parent)
72 : QAbstractNetworkCache(*new QNetworkDiskCachePrivate, parent)
73{
74}
75
76/*!
77 Destroys the cache object. This does not clear the disk cache.
78 */
79QNetworkDiskCache::~QNetworkDiskCache()
80{
81 Q_D(QNetworkDiskCache);
82 qDeleteAll(c: d->inserting);
83}
84
85/*!
86 Returns the location where cached files will be stored.
87*/
88QString QNetworkDiskCache::cacheDirectory() const
89{
90 Q_D(const QNetworkDiskCache);
91 return d->cacheDirectory;
92}
93
94/*!
95 Sets the directory where cached files will be stored to \a cacheDir
96
97 QNetworkDiskCache will create this directory if it does not exists.
98
99 Prepared cache items will be stored in the new cache directory when
100 they are inserted.
101
102 \sa QStandardPaths::CacheLocation
103*/
104void QNetworkDiskCache::setCacheDirectory(const QString &cacheDir)
105{
106#if defined(QNETWORKDISKCACHE_DEBUG)
107 qDebug() << "QNetworkDiskCache::setCacheDirectory()" << cacheDir;
108#endif
109 Q_D(QNetworkDiskCache);
110 if (cacheDir.isEmpty())
111 return;
112 d->cacheDirectory = cacheDir;
113 QDir dir(d->cacheDirectory);
114 d->cacheDirectory = dir.absolutePath();
115 if (!d->cacheDirectory.endsWith(c: u'/'))
116 d->cacheDirectory += u'/';
117
118 d->dataDirectory = d->cacheDirectory + DATA_DIR + QString::number(CACHE_VERSION) + u'/';
119 d->prepareLayout();
120}
121
122/*!
123 \reimp
124*/
125qint64 QNetworkDiskCache::cacheSize() const
126{
127#if defined(QNETWORKDISKCACHE_DEBUG)
128 qDebug("QNetworkDiskCache::cacheSize()");
129#endif
130 Q_D(const QNetworkDiskCache);
131 if (d->cacheDirectory.isEmpty())
132 return 0;
133 if (d->currentCacheSize < 0) {
134 QNetworkDiskCache *that = const_cast<QNetworkDiskCache*>(this);
135 that->d_func()->currentCacheSize = that->expire();
136 }
137 return d->currentCacheSize;
138}
139
140/*!
141 \reimp
142*/
143QIODevice *QNetworkDiskCache::prepare(const QNetworkCacheMetaData &metaData)
144{
145#if defined(QNETWORKDISKCACHE_DEBUG)
146 qDebug() << "QNetworkDiskCache::prepare()" << metaData.url();
147#endif
148 Q_D(QNetworkDiskCache);
149 if (!metaData.isValid() || !metaData.url().isValid() || !metaData.saveToDisk())
150 return nullptr;
151
152 if (d->cacheDirectory.isEmpty()) {
153 qWarning(msg: "QNetworkDiskCache::prepare() The cache directory is not set");
154 return nullptr;
155 }
156
157 const auto sizeValue = metaData.headers().value(name: QHttpHeaders::WellKnownHeader::ContentLength);
158 const qint64 size = sizeValue.toLongLong();
159 if (size > (maximumCacheSize() * 3)/4)
160 return nullptr;
161
162 std::unique_ptr<QCacheItem> cacheItem = std::make_unique<QCacheItem>();
163 cacheItem->metaData = metaData;
164
165 QIODevice *device = nullptr;
166 if (cacheItem->canCompress()) {
167 cacheItem->data.open(openMode: QBuffer::ReadWrite);
168 device = &(cacheItem->data);
169 } else {
170 QString fileName = d->cacheFileName(url: cacheItem->metaData.url());
171 cacheItem->file = new(std::nothrow) QSaveFile(fileName, &cacheItem->data);
172 if (!cacheItem->file || !cacheItem->file->open(flags: QFileDevice::WriteOnly)) {
173 qWarning(msg: "QNetworkDiskCache::prepare() unable to open temporary file");
174 cacheItem.reset();
175 return nullptr;
176 }
177 cacheItem->writeHeader(device: cacheItem->file);
178 device = cacheItem->file;
179 }
180 d->inserting[device] = cacheItem.release();
181 return device;
182}
183
184/*!
185 \reimp
186*/
187void QNetworkDiskCache::insert(QIODevice *device)
188{
189#if defined(QNETWORKDISKCACHE_DEBUG)
190 qDebug() << "QNetworkDiskCache::insert()" << device;
191#endif
192 Q_D(QNetworkDiskCache);
193 const auto it = d->inserting.constFind(key: device);
194 if (Q_UNLIKELY(it == d->inserting.cend())) {
195 qWarning() << "QNetworkDiskCache::insert() called on a device we don't know about" << device;
196 return;
197 }
198
199 d->storeItem(item: it.value());
200 delete it.value();
201 d->inserting.erase(it);
202}
203
204
205/*!
206 Create subdirectories and other housekeeping on the filesystem.
207 Prevents too many files from being present in any single directory.
208*/
209void QNetworkDiskCachePrivate::prepareLayout()
210{
211 QDir helper;
212
213 //Create directory and subdirectories 0-F
214 helper.mkpath(dirPath: dataDirectory);
215 for (uint i = 0; i < 16 ; i++) {
216 QString str = QString::number(i, base: 16);
217 QString subdir = dataDirectory + str;
218 helper.mkdir(dirName: subdir);
219 }
220}
221
222
223void QNetworkDiskCachePrivate::storeItem(QCacheItem *cacheItem)
224{
225 Q_Q(QNetworkDiskCache);
226 Q_ASSERT(cacheItem->metaData.saveToDisk());
227
228 QString fileName = cacheFileName(url: cacheItem->metaData.url());
229 Q_ASSERT(!fileName.isEmpty());
230
231 if (QFile::exists(fileName)) {
232 if (!removeFile(file: fileName)) {
233 qWarning() << "QNetworkDiskCache: couldn't remove the cache file " << fileName;
234 return;
235 }
236 }
237
238 currentCacheSize = q->expire();
239 if (!cacheItem->file) {
240 cacheItem->file = new QSaveFile(fileName, &cacheItem->data);
241 if (cacheItem->file->open(flags: QFileDevice::WriteOnly)) {
242 cacheItem->writeHeader(device: cacheItem->file);
243 cacheItem->writeCompressedData(device: cacheItem->file);
244 }
245 }
246
247 if (cacheItem->file
248 && cacheItem->file->isOpen()
249 && cacheItem->file->error() == QFileDevice::NoError) {
250 // We have to call size() here instead of inside the if-body because
251 // commit() invalidates the file-engine, and size() will create a new
252 // one, pointing at an empty filename.
253 qint64 size = cacheItem->file->size();
254 if (cacheItem->file->commit())
255 currentCacheSize += size;
256 // Delete and unset the QSaveFile, it's invalid now.
257 delete std::exchange(obj&: cacheItem->file, new_val: nullptr);
258 }
259 if (cacheItem->metaData.url() == lastItem.metaData.url())
260 lastItem.reset();
261}
262
263/*!
264 \reimp
265*/
266bool QNetworkDiskCache::remove(const QUrl &url)
267{
268#if defined(QNETWORKDISKCACHE_DEBUG)
269 qDebug() << "QNetworkDiskCache::remove()" << url;
270#endif
271 Q_D(QNetworkDiskCache);
272
273 // remove is also used to cancel insertions, not a common operation
274 for (auto it = d->inserting.cbegin(), end = d->inserting.cend(); it != end; ++it) {
275 QCacheItem *item = it.value();
276 if (item && item->metaData.url() == url) {
277 delete item;
278 d->inserting.erase(it);
279 return true;
280 }
281 }
282
283 if (d->lastItem.metaData.url() == url)
284 d->lastItem.reset();
285 return d->removeFile(file: d->cacheFileName(url));
286}
287
288/*!
289 Put all of the misc file removing into one function to be extra safe
290 */
291bool QNetworkDiskCachePrivate::removeFile(const QString &file)
292{
293#if defined(QNETWORKDISKCACHE_DEBUG)
294 qDebug() << "QNetworkDiskCache::removFile()" << file;
295#endif
296 if (file.isEmpty())
297 return false;
298 QFileInfo info(file);
299 QString fileName = info.fileName();
300 if (!fileName.endsWith(CACHE_POSTFIX))
301 return false;
302 qint64 size = info.size();
303 if (QFile::remove(fileName: file)) {
304 currentCacheSize -= size;
305 return true;
306 }
307 return false;
308}
309
310/*!
311 \reimp
312*/
313QNetworkCacheMetaData QNetworkDiskCache::metaData(const QUrl &url)
314{
315#if defined(QNETWORKDISKCACHE_DEBUG)
316 qDebug() << "QNetworkDiskCache::metaData()" << url;
317#endif
318 Q_D(QNetworkDiskCache);
319 if (d->lastItem.metaData.url() == url)
320 return d->lastItem.metaData;
321 return fileMetaData(fileName: d->cacheFileName(url));
322}
323
324/*!
325 Returns the QNetworkCacheMetaData for the cache file \a fileName.
326
327 If \a fileName is not a cache file QNetworkCacheMetaData will be invalid.
328 */
329QNetworkCacheMetaData QNetworkDiskCache::fileMetaData(const QString &fileName) const
330{
331#if defined(QNETWORKDISKCACHE_DEBUG)
332 qDebug() << "QNetworkDiskCache::fileMetaData()" << fileName;
333#endif
334 Q_D(const QNetworkDiskCache);
335 QFile file(fileName);
336 if (!file.open(flags: QFile::ReadOnly))
337 return QNetworkCacheMetaData();
338 if (!d->lastItem.read(device: &file, readData: false)) {
339 file.close();
340 QNetworkDiskCachePrivate *that = const_cast<QNetworkDiskCachePrivate*>(d);
341 that->removeFile(file: fileName);
342 }
343 return d->lastItem.metaData;
344}
345
346/*!
347 \reimp
348*/
349QIODevice *QNetworkDiskCache::data(const QUrl &url)
350{
351#if defined(QNETWORKDISKCACHE_DEBUG)
352 qDebug() << "QNetworkDiskCache::data()" << url;
353#endif
354 Q_D(QNetworkDiskCache);
355 std::unique_ptr<QBuffer> buffer;
356 if (!url.isValid())
357 return nullptr;
358 if (d->lastItem.metaData.url() == url && d->lastItem.data.isOpen()) {
359 buffer.reset(p: new QBuffer);
360 buffer->setData(d->lastItem.data.data());
361 } else {
362 QScopedPointer<QFile> file(new QFile(d->cacheFileName(url)));
363 if (!file->open(flags: QFile::ReadOnly | QIODevice::Unbuffered))
364 return nullptr;
365
366 if (!d->lastItem.read(device: file.data(), readData: true)) {
367 file->close();
368 remove(url);
369 return nullptr;
370 }
371 if (d->lastItem.data.isOpen()) {
372 // compressed
373 buffer.reset(p: new QBuffer);
374 buffer->setData(d->lastItem.data.data());
375 } else {
376 buffer.reset(p: new QBuffer);
377 buffer->setData(file->readAll());
378 }
379 }
380 buffer->open(openMode: QBuffer::ReadOnly);
381 return buffer.release();
382}
383
384/*!
385 \reimp
386*/
387void QNetworkDiskCache::updateMetaData(const QNetworkCacheMetaData &metaData)
388{
389#if defined(QNETWORKDISKCACHE_DEBUG)
390 qDebug() << "QNetworkDiskCache::updateMetaData()" << metaData.url();
391#endif
392 QUrl url = metaData.url();
393 QIODevice *oldDevice = data(url);
394 if (!oldDevice) {
395#if defined(QNETWORKDISKCACHE_DEBUG)
396 qDebug("QNetworkDiskCache::updateMetaData(), no device!");
397#endif
398 return;
399 }
400
401 QIODevice *newDevice = prepare(metaData);
402 if (!newDevice) {
403#if defined(QNETWORKDISKCACHE_DEBUG)
404 qDebug() << "QNetworkDiskCache::updateMetaData(), no new device!" << url;
405#endif
406 return;
407 }
408 char data[1024];
409 while (!oldDevice->atEnd()) {
410 qint64 s = oldDevice->read(data, maxlen: 1024);
411 newDevice->write(data, len: s);
412 }
413 delete oldDevice;
414 insert(device: newDevice);
415}
416
417/*!
418 Returns the current maximum size for the disk cache.
419
420 \sa setMaximumCacheSize()
421 */
422qint64 QNetworkDiskCache::maximumCacheSize() const
423{
424 Q_D(const QNetworkDiskCache);
425 return d->maximumCacheSize;
426}
427
428/*!
429 Sets the maximum size of the disk cache to be \a size.
430
431 If the new size is smaller then the current cache size then the cache will call expire().
432
433 \sa maximumCacheSize()
434 */
435void QNetworkDiskCache::setMaximumCacheSize(qint64 size)
436{
437 Q_D(QNetworkDiskCache);
438 bool expireCache = (size < d->maximumCacheSize);
439 d->maximumCacheSize = size;
440 if (expireCache)
441 d->currentCacheSize = expire();
442}
443
444/*!
445 Cleans the cache so that its size is under the maximum cache size.
446 Returns the current size of the cache.
447
448 When the current size of the cache is greater than the maximumCacheSize()
449 older cache files are removed until the total size is less then 90% of
450 maximumCacheSize() starting with the oldest ones first using the file
451 creation date to determine how old a cache file is.
452
453 Subclasses can reimplement this function to change the order that cache
454 files are removed taking into account information in the application
455 knows about that QNetworkDiskCache does not, for example the number of times
456 a cache is accessed.
457
458 \note cacheSize() calls expire if the current cache size is unknown.
459
460 \sa maximumCacheSize(), fileMetaData()
461 */
462qint64 QNetworkDiskCache::expire()
463{
464 Q_D(QNetworkDiskCache);
465 if (d->currentCacheSize >= 0 && d->currentCacheSize < maximumCacheSize())
466 return d->currentCacheSize;
467
468 if (cacheDirectory().isEmpty()) {
469 qWarning(msg: "QNetworkDiskCache::expire() The cache directory is not set");
470 return 0;
471 }
472
473 // close file handle to prevent "in use" error when QFile::remove() is called
474 d->lastItem.reset();
475
476 struct CacheItem
477 {
478 std::chrono::milliseconds msecs;
479 QString path;
480 qint64 size = 0;
481 };
482 std::vector<CacheItem> cacheItems;
483 qint64 totalSize = 0;
484 using F = QDirListing::IteratorFlag;
485 for (const auto &dirEntry : QDirListing(cacheDirectory(), F::FilesOnly | F::Recursive)) {
486 if (!dirEntry.fileName().endsWith(CACHE_POSTFIX))
487 continue;
488
489 const QFileInfo &info = dirEntry.fileInfo();
490 QDateTime fileTime = info.birthTime(tz: QTimeZone::UTC);
491 if (!fileTime.isValid())
492 fileTime = info.metadataChangeTime(tz: QTimeZone::UTC);
493 const std::chrono::milliseconds msecs{fileTime.toMSecsSinceEpoch()};
494 const qint64 size = info.size();
495 cacheItems.push_back(x: CacheItem{.msecs: msecs, .path: info.filePath(), .size: size});
496 totalSize += size;
497 }
498
499 const qint64 goal = (maximumCacheSize() * 9) / 10;
500 if (totalSize < goal)
501 return totalSize; // Nothing to do
502
503 auto byFileTime = [&](const auto &a, const auto &b) { return a.msecs < b.msecs; };
504 std::sort(first: cacheItems.begin(), last: cacheItems.end(), comp: byFileTime);
505
506 [[maybe_unused]] int removedFiles = 0; // used under QNETWORKDISKCACHE_DEBUG
507 for (const CacheItem &cached : cacheItems) {
508 QFile::remove(fileName: cached.path);
509 ++removedFiles;
510 totalSize -= cached.size;
511 if (totalSize < goal)
512 break;
513 }
514#if defined(QNETWORKDISKCACHE_DEBUG)
515 if (removedFiles > 0) {
516 qDebug() << "QNetworkDiskCache::expire()"
517 << "Removed:" << removedFiles
518 << "Kept:" << cacheItems.count() - removedFiles;
519 }
520#endif
521 return totalSize;
522}
523
524/*!
525 \reimp
526*/
527void QNetworkDiskCache::clear()
528{
529#if defined(QNETWORKDISKCACHE_DEBUG)
530 qDebug("QNetworkDiskCache::clear()");
531#endif
532 Q_D(QNetworkDiskCache);
533 qint64 size = d->maximumCacheSize;
534 d->maximumCacheSize = 0;
535 d->currentCacheSize = expire();
536 d->maximumCacheSize = size;
537}
538
539/*!
540 Given a URL, generates a unique enough filename (and subdirectory)
541 */
542QString QNetworkDiskCachePrivate::uniqueFileName(const QUrl &url)
543{
544 QUrl cleanUrl = url;
545 cleanUrl.setPassword(password: QString());
546 cleanUrl.setFragment(fragment: QString());
547
548 const QByteArray hash = QCryptographicHash::hash(data: cleanUrl.toEncoded(), method: QCryptographicHash::Sha1);
549 // convert sha1 to base36 form and return first 8 bytes for use as string
550 const QByteArray id = QByteArray::number(*(qlonglong*)hash.data(), base: 36).left(n: 8);
551 // generates <one-char subdir>/<8-char filename.d>
552 uint code = (uint)id.at(i: id.size()-1) % 16;
553 QString pathFragment = QString::number(code, base: 16) + u'/' + QLatin1StringView(id) + CACHE_POSTFIX;
554
555 return pathFragment;
556}
557
558/*!
559 Generates fully qualified path of cached resource from a URL.
560 */
561QString QNetworkDiskCachePrivate::cacheFileName(const QUrl &url) const
562{
563 if (!url.isValid())
564 return QString();
565
566 QString fullpath = dataDirectory + uniqueFileName(url);
567 return fullpath;
568}
569
570/*!
571 We compress small text and JavaScript files.
572 */
573bool QCacheItem::canCompress() const
574{
575 const auto h = metaData.headers();
576
577 const auto sizeValue = h.value(name: QHttpHeaders::WellKnownHeader::ContentLength);
578 if (sizeValue.empty())
579 return false;
580
581 qint64 size = sizeValue.toLongLong();
582 if (size > MAX_COMPRESSION_SIZE)
583 return false;
584
585 const auto type = h.value(name: QHttpHeaders::WellKnownHeader::ContentType);
586 if (!type.empty())
587 return false;
588
589 if (!type.startsWith(other: "text/")
590 && !(type.startsWith(other: "application/")
591 && (type.endsWith(other: "javascript") || type.endsWith(other: "ecmascript")))) {
592 return false;
593 }
594
595 return true;
596}
597
598enum
599{
600 CacheMagic = 0xe8,
601 CurrentCacheVersion = CACHE_VERSION
602};
603
604void QCacheItem::writeHeader(QFileDevice *device) const
605{
606 QDataStream out(device);
607
608 out << qint32(CacheMagic);
609 out << qint32(CurrentCacheVersion);
610 out << static_cast<qint32>(out.version());
611 out << metaData;
612 bool compressed = canCompress();
613 out << compressed;
614}
615
616void QCacheItem::writeCompressedData(QFileDevice *device) const
617{
618 QDataStream out(device);
619
620 out << qCompress(data: data.data());
621}
622
623/*!
624 Returns \c false if the file is a cache file,
625 but is an older version and should be removed otherwise true.
626 */
627bool QCacheItem::read(QFileDevice *device, bool readData)
628{
629 reset();
630
631 QDataStream in(device);
632
633 qint32 marker;
634 qint32 v;
635 in >> marker;
636 in >> v;
637 if (marker != CacheMagic)
638 return true;
639
640 // If the cache magic is correct, but the version is not we should remove it
641 if (v != CurrentCacheVersion)
642 return false;
643
644 qint32 streamVersion;
645 in >> streamVersion;
646 // Default stream version is also the highest we can handle
647 if (streamVersion > in.version())
648 return false;
649 in.setVersion(streamVersion);
650
651 bool compressed;
652 QByteArray dataBA;
653 in >> metaData;
654 in >> compressed;
655 if (readData && compressed) {
656 in >> dataBA;
657 data.setData(qUncompress(data: dataBA));
658 data.open(openMode: QBuffer::ReadOnly);
659 }
660
661 // quick and dirty check if metadata's URL field and the file's name are in synch
662 QString expectedFilename = QNetworkDiskCachePrivate::uniqueFileName(url: metaData.url());
663 if (!device->fileName().endsWith(s: expectedFilename))
664 return false;
665
666 return metaData.isValid() && !metaData.headers().isEmpty();
667}
668
669QT_END_NAMESPACE
670
671#include "moc_qnetworkdiskcache.cpp"
672

source code of qtbase/src/network/access/qnetworkdiskcache.cpp