1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4//#define QNETWORKDISKCACHE_DEBUG
5
6
7#include "qnetworkdiskcache.h"
8#include "qnetworkdiskcache_p.h"
9#include "QtCore/qscopedpointer.h"
10
11#include <qfile.h>
12#include <qdir.h>
13#include <qdatastream.h>
14#include <qdatetime.h>
15#include <qdiriterator.h>
16#include <qurl.h>
17#include <qcryptographichash.h>
18#include <qdebug.h>
19
20#include <memory>
21
22#define CACHE_POSTFIX ".d"_L1
23#define CACHE_VERSION 8
24#define DATA_DIR "data"_L1
25
26#define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)
27
28QT_BEGIN_NAMESPACE
29
30using namespace Qt::StringLiterals;
31
32/*!
33 \class QNetworkDiskCache
34 \since 4.5
35 \inmodule QtNetwork
36
37 \brief The QNetworkDiskCache class provides a very basic disk cache.
38
39 QNetworkDiskCache stores each url in its own file inside of the
40 cacheDirectory using QDataStream. Files with a text MimeType
41 are compressed using qCompress. Data is written to disk only in insert()
42 and updateMetaData().
43
44 Currently you cannot share the same cache files with more than
45 one disk cache.
46
47 QNetworkDiskCache by default limits the amount of space that the cache will
48 use on the system to 50MB.
49
50 Note you have to set the cache directory before it will work.
51
52 A network disk cache can be enabled by:
53
54 \snippet code/src_network_access_qnetworkdiskcache.cpp 0
55
56 When sending requests, to control the preference of when to use the cache
57 and when to use the network, consider the following:
58
59 \snippet code/src_network_access_qnetworkdiskcache.cpp 1
60
61 To check whether the response came from the cache or from the network, the
62 following can be applied:
63
64 \snippet code/src_network_access_qnetworkdiskcache.cpp 2
65*/
66
67/*!
68 Creates a new disk cache. The \a parent argument is passed to
69 QAbstractNetworkCache's constructor.
70 */
71QNetworkDiskCache::QNetworkDiskCache(QObject *parent)
72 : QAbstractNetworkCache(*new QNetworkDiskCachePrivate, parent)
73{
74}
75
76/*!
77 Destroys the cache object. This does not clear the disk cache.
78 */
79QNetworkDiskCache::~QNetworkDiskCache()
80{
81 Q_D(QNetworkDiskCache);
82 qDeleteAll(c: d->inserting);
83}
84
85/*!
86 Returns the location where cached files will be stored.
87*/
88QString QNetworkDiskCache::cacheDirectory() const
89{
90 Q_D(const QNetworkDiskCache);
91 return d->cacheDirectory;
92}
93
94/*!
95 Sets the directory where cached files will be stored to \a cacheDir
96
97 QNetworkDiskCache will create this directory if it does not exists.
98
99 Prepared cache items will be stored in the new cache directory when
100 they are inserted.
101
102 \sa QStandardPaths::CacheLocation
103*/
104void QNetworkDiskCache::setCacheDirectory(const QString &cacheDir)
105{
106#if defined(QNETWORKDISKCACHE_DEBUG)
107 qDebug() << "QNetworkDiskCache::setCacheDirectory()" << cacheDir;
108#endif
109 Q_D(QNetworkDiskCache);
110 if (cacheDir.isEmpty())
111 return;
112 d->cacheDirectory = cacheDir;
113 QDir dir(d->cacheDirectory);
114 d->cacheDirectory = dir.absolutePath();
115 if (!d->cacheDirectory.endsWith(c: u'/'))
116 d->cacheDirectory += u'/';
117
118 d->dataDirectory = d->cacheDirectory + DATA_DIR + QString::number(CACHE_VERSION) + u'/';
119 d->prepareLayout();
120}
121
122/*!
123 \reimp
124*/
125qint64 QNetworkDiskCache::cacheSize() const
126{
127#if defined(QNETWORKDISKCACHE_DEBUG)
128 qDebug("QNetworkDiskCache::cacheSize()");
129#endif
130 Q_D(const QNetworkDiskCache);
131 if (d->cacheDirectory.isEmpty())
132 return 0;
133 if (d->currentCacheSize < 0) {
134 QNetworkDiskCache *that = const_cast<QNetworkDiskCache*>(this);
135 that->d_func()->currentCacheSize = that->expire();
136 }
137 return d->currentCacheSize;
138}
139
140/*!
141 \reimp
142*/
143QIODevice *QNetworkDiskCache::prepare(const QNetworkCacheMetaData &metaData)
144{
145#if defined(QNETWORKDISKCACHE_DEBUG)
146 qDebug() << "QNetworkDiskCache::prepare()" << metaData.url();
147#endif
148 Q_D(QNetworkDiskCache);
149 if (!metaData.isValid() || !metaData.url().isValid() || !metaData.saveToDisk())
150 return nullptr;
151
152 if (d->cacheDirectory.isEmpty()) {
153 qWarning(msg: "QNetworkDiskCache::prepare() The cache directory is not set");
154 return nullptr;
155 }
156
157 const auto headers = metaData.rawHeaders();
158 for (const auto &header : headers) {
159 if (header.first.compare(a: "content-length", cs: Qt::CaseInsensitive) == 0) {
160 const qint64 size = header.second.toLongLong();
161 if (size > (maximumCacheSize() * 3)/4)
162 return nullptr;
163 break;
164 }
165 }
166 std::unique_ptr<QCacheItem> cacheItem = std::make_unique<QCacheItem>();
167 cacheItem->metaData = metaData;
168
169 QIODevice *device = nullptr;
170 if (cacheItem->canCompress()) {
171 cacheItem->data.open(openMode: QBuffer::ReadWrite);
172 device = &(cacheItem->data);
173 } else {
174 QString fileName = d->cacheFileName(url: cacheItem->metaData.url());
175 QT_TRY {
176 cacheItem->file = new QSaveFile(fileName, &cacheItem->data);
177 } QT_CATCH(...) {
178 cacheItem->file = nullptr;
179 }
180 if (!cacheItem->file || !cacheItem->file->open(flags: QFileDevice::WriteOnly)) {
181 qWarning(msg: "QNetworkDiskCache::prepare() unable to open temporary file");
182 cacheItem.reset();
183 return nullptr;
184 }
185 cacheItem->writeHeader(device: cacheItem->file);
186 device = cacheItem->file;
187 }
188 d->inserting[device] = cacheItem.release();
189 return device;
190}
191
192/*!
193 \reimp
194*/
195void QNetworkDiskCache::insert(QIODevice *device)
196{
197#if defined(QNETWORKDISKCACHE_DEBUG)
198 qDebug() << "QNetworkDiskCache::insert()" << device;
199#endif
200 Q_D(QNetworkDiskCache);
201 const auto it = d->inserting.constFind(key: device);
202 if (Q_UNLIKELY(it == d->inserting.cend())) {
203 qWarning() << "QNetworkDiskCache::insert() called on a device we don't know about" << device;
204 return;
205 }
206
207 d->storeItem(item: it.value());
208 delete it.value();
209 d->inserting.erase(it);
210}
211
212
213/*!
214 Create subdirectories and other housekeeping on the filesystem.
215 Prevents too many files from being present in any single directory.
216*/
217void QNetworkDiskCachePrivate::prepareLayout()
218{
219 QDir helper;
220
221 //Create directory and subdirectories 0-F
222 helper.mkpath(dirPath: dataDirectory);
223 for (uint i = 0; i < 16 ; i++) {
224 QString str = QString::number(i, base: 16);
225 QString subdir = dataDirectory + str;
226 helper.mkdir(dirName: subdir);
227 }
228}
229
230
231void QNetworkDiskCachePrivate::storeItem(QCacheItem *cacheItem)
232{
233 Q_Q(QNetworkDiskCache);
234 Q_ASSERT(cacheItem->metaData.saveToDisk());
235
236 QString fileName = cacheFileName(url: cacheItem->metaData.url());
237 Q_ASSERT(!fileName.isEmpty());
238
239 if (QFile::exists(fileName)) {
240 if (!removeFile(file: fileName)) {
241 qWarning() << "QNetworkDiskCache: couldn't remove the cache file " << fileName;
242 return;
243 }
244 }
245
246 currentCacheSize = q->expire();
247 if (!cacheItem->file) {
248 cacheItem->file = new QSaveFile(fileName, &cacheItem->data);
249 if (cacheItem->file->open(flags: QFileDevice::WriteOnly)) {
250 cacheItem->writeHeader(device: cacheItem->file);
251 cacheItem->writeCompressedData(device: cacheItem->file);
252 }
253 }
254
255 if (cacheItem->file
256 && cacheItem->file->isOpen()
257 && cacheItem->file->error() == QFileDevice::NoError) {
258 // We have to call size() here instead of inside the if-body because
259 // commit() invalidates the file-engine, and size() will create a new
260 // one, pointing at an empty filename.
261 qint64 size = cacheItem->file->size();
262 if (cacheItem->file->commit())
263 currentCacheSize += size;
264 // Delete and unset the QSaveFile, it's invalid now.
265 delete std::exchange(obj&: cacheItem->file, new_val: nullptr);
266 }
267 if (cacheItem->metaData.url() == lastItem.metaData.url())
268 lastItem.reset();
269}
270
271/*!
272 \reimp
273*/
274bool QNetworkDiskCache::remove(const QUrl &url)
275{
276#if defined(QNETWORKDISKCACHE_DEBUG)
277 qDebug() << "QNetworkDiskCache::remove()" << url;
278#endif
279 Q_D(QNetworkDiskCache);
280
281 // remove is also used to cancel insertions, not a common operation
282 for (auto it = d->inserting.cbegin(), end = d->inserting.cend(); it != end; ++it) {
283 QCacheItem *item = it.value();
284 if (item && item->metaData.url() == url) {
285 delete item;
286 d->inserting.erase(it);
287 return true;
288 }
289 }
290
291 if (d->lastItem.metaData.url() == url)
292 d->lastItem.reset();
293 return d->removeFile(file: d->cacheFileName(url));
294}
295
296/*!
297 Put all of the misc file removing into one function to be extra safe
298 */
299bool QNetworkDiskCachePrivate::removeFile(const QString &file)
300{
301#if defined(QNETWORKDISKCACHE_DEBUG)
302 qDebug() << "QNetworkDiskCache::removFile()" << file;
303#endif
304 if (file.isEmpty())
305 return false;
306 QFileInfo info(file);
307 QString fileName = info.fileName();
308 if (!fileName.endsWith(CACHE_POSTFIX))
309 return false;
310 qint64 size = info.size();
311 if (QFile::remove(fileName: file)) {
312 currentCacheSize -= size;
313 return true;
314 }
315 return false;
316}
317
318/*!
319 \reimp
320*/
321QNetworkCacheMetaData QNetworkDiskCache::metaData(const QUrl &url)
322{
323#if defined(QNETWORKDISKCACHE_DEBUG)
324 qDebug() << "QNetworkDiskCache::metaData()" << url;
325#endif
326 Q_D(QNetworkDiskCache);
327 if (d->lastItem.metaData.url() == url)
328 return d->lastItem.metaData;
329 return fileMetaData(fileName: d->cacheFileName(url));
330}
331
332/*!
333 Returns the QNetworkCacheMetaData for the cache file \a fileName.
334
335 If \a fileName is not a cache file QNetworkCacheMetaData will be invalid.
336 */
337QNetworkCacheMetaData QNetworkDiskCache::fileMetaData(const QString &fileName) const
338{
339#if defined(QNETWORKDISKCACHE_DEBUG)
340 qDebug() << "QNetworkDiskCache::fileMetaData()" << fileName;
341#endif
342 Q_D(const QNetworkDiskCache);
343 QFile file(fileName);
344 if (!file.open(flags: QFile::ReadOnly))
345 return QNetworkCacheMetaData();
346 if (!d->lastItem.read(device: &file, readData: false)) {
347 file.close();
348 QNetworkDiskCachePrivate *that = const_cast<QNetworkDiskCachePrivate*>(d);
349 that->removeFile(file: fileName);
350 }
351 return d->lastItem.metaData;
352}
353
354/*!
355 \reimp
356*/
357QIODevice *QNetworkDiskCache::data(const QUrl &url)
358{
359#if defined(QNETWORKDISKCACHE_DEBUG)
360 qDebug() << "QNetworkDiskCache::data()" << url;
361#endif
362 Q_D(QNetworkDiskCache);
363 std::unique_ptr<QBuffer> buffer;
364 if (!url.isValid())
365 return nullptr;
366 if (d->lastItem.metaData.url() == url && d->lastItem.data.isOpen()) {
367 buffer.reset(p: new QBuffer);
368 buffer->setData(d->lastItem.data.data());
369 } else {
370 QScopedPointer<QFile> file(new QFile(d->cacheFileName(url)));
371 if (!file->open(flags: QFile::ReadOnly | QIODevice::Unbuffered))
372 return nullptr;
373
374 if (!d->lastItem.read(device: file.data(), readData: true)) {
375 file->close();
376 remove(url);
377 return nullptr;
378 }
379 if (d->lastItem.data.isOpen()) {
380 // compressed
381 buffer.reset(p: new QBuffer);
382 buffer->setData(d->lastItem.data.data());
383 } else {
384 buffer.reset(p: new QBuffer);
385 buffer->setData(file->readAll());
386 }
387 }
388 buffer->open(openMode: QBuffer::ReadOnly);
389 return buffer.release();
390}
391
392/*!
393 \reimp
394*/
395void QNetworkDiskCache::updateMetaData(const QNetworkCacheMetaData &metaData)
396{
397#if defined(QNETWORKDISKCACHE_DEBUG)
398 qDebug() << "QNetworkDiskCache::updateMetaData()" << metaData.url();
399#endif
400 QUrl url = metaData.url();
401 QIODevice *oldDevice = data(url);
402 if (!oldDevice) {
403#if defined(QNETWORKDISKCACHE_DEBUG)
404 qDebug("QNetworkDiskCache::updateMetaData(), no device!");
405#endif
406 return;
407 }
408
409 QIODevice *newDevice = prepare(metaData);
410 if (!newDevice) {
411#if defined(QNETWORKDISKCACHE_DEBUG)
412 qDebug() << "QNetworkDiskCache::updateMetaData(), no new device!" << url;
413#endif
414 return;
415 }
416 char data[1024];
417 while (!oldDevice->atEnd()) {
418 qint64 s = oldDevice->read(data, maxlen: 1024);
419 newDevice->write(data, len: s);
420 }
421 delete oldDevice;
422 insert(device: newDevice);
423}
424
425/*!
426 Returns the current maximum size for the disk cache.
427
428 \sa setMaximumCacheSize()
429 */
430qint64 QNetworkDiskCache::maximumCacheSize() const
431{
432 Q_D(const QNetworkDiskCache);
433 return d->maximumCacheSize;
434}
435
436/*!
437 Sets the maximum size of the disk cache to be \a size.
438
439 If the new size is smaller then the current cache size then the cache will call expire().
440
441 \sa maximumCacheSize()
442 */
443void QNetworkDiskCache::setMaximumCacheSize(qint64 size)
444{
445 Q_D(QNetworkDiskCache);
446 bool expireCache = (size < d->maximumCacheSize);
447 d->maximumCacheSize = size;
448 if (expireCache)
449 d->currentCacheSize = expire();
450}
451
452/*!
453 Cleans the cache so that its size is under the maximum cache size.
454 Returns the current size of the cache.
455
456 When the current size of the cache is greater than the maximumCacheSize()
457 older cache files are removed until the total size is less then 90% of
458 maximumCacheSize() starting with the oldest ones first using the file
459 creation date to determine how old a cache file is.
460
461 Subclasses can reimplement this function to change the order that cache
462 files are removed taking into account information in the application
463 knows about that QNetworkDiskCache does not, for example the number of times
464 a cache is accessed.
465
466 \note cacheSize() calls expire if the current cache size is unknown.
467
468 \sa maximumCacheSize(), fileMetaData()
469 */
470qint64 QNetworkDiskCache::expire()
471{
472 Q_D(QNetworkDiskCache);
473 if (d->currentCacheSize >= 0 && d->currentCacheSize < maximumCacheSize())
474 return d->currentCacheSize;
475
476 if (cacheDirectory().isEmpty()) {
477 qWarning(msg: "QNetworkDiskCache::expire() The cache directory is not set");
478 return 0;
479 }
480
481 // close file handle to prevent "in use" error when QFile::remove() is called
482 d->lastItem.reset();
483
484 const QDir::Filters filters = QDir::AllDirs | QDir:: Files | QDir::NoDotAndDotDot;
485 QDirIterator it(cacheDirectory(), filters, QDirIterator::Subdirectories);
486
487 struct CacheItem
488 {
489 std::chrono::milliseconds msecs;
490 QString path;
491 qint64 size = 0;
492 };
493 std::vector<CacheItem> cacheItems;
494 qint64 totalSize = 0;
495 while (it.hasNext()) {
496 QFileInfo info = it.nextFileInfo();
497 if (!info.fileName().endsWith(CACHE_POSTFIX))
498 continue;
499
500 QDateTime fileTime = info.birthTime(tz: QTimeZone::UTC);
501 if (!fileTime.isValid())
502 fileTime = info.metadataChangeTime(tz: QTimeZone::UTC);
503 const std::chrono::milliseconds msecs{fileTime.toMSecsSinceEpoch()};
504 const qint64 size = info.size();
505 cacheItems.push_back(x: CacheItem{.msecs: msecs, .path: info.filePath(), .size: size});
506 totalSize += size;
507 }
508
509 const qint64 goal = (maximumCacheSize() * 9) / 10;
510 if (totalSize < goal)
511 return totalSize; // Nothing to do
512
513 auto byFileTime = [&](const auto &a, const auto &b) { return a.msecs < b.msecs; };
514 std::sort(first: cacheItems.begin(), last: cacheItems.end(), comp: byFileTime);
515
516 [[maybe_unused]] int removedFiles = 0; // used under QNETWORKDISKCACHE_DEBUG
517 for (const CacheItem &cached : cacheItems) {
518 QFile::remove(fileName: cached.path);
519 ++removedFiles;
520 totalSize -= cached.size;
521 if (totalSize < goal)
522 break;
523 }
524#if defined(QNETWORKDISKCACHE_DEBUG)
525 if (removedFiles > 0) {
526 qDebug() << "QNetworkDiskCache::expire()"
527 << "Removed:" << removedFiles
528 << "Kept:" << cacheItems.count() - removedFiles;
529 }
530#endif
531 return totalSize;
532}
533
534/*!
535 \reimp
536*/
537void QNetworkDiskCache::clear()
538{
539#if defined(QNETWORKDISKCACHE_DEBUG)
540 qDebug("QNetworkDiskCache::clear()");
541#endif
542 Q_D(QNetworkDiskCache);
543 qint64 size = d->maximumCacheSize;
544 d->maximumCacheSize = 0;
545 d->currentCacheSize = expire();
546 d->maximumCacheSize = size;
547}
548
549/*!
550 Given a URL, generates a unique enough filename (and subdirectory)
551 */
552QString QNetworkDiskCachePrivate::uniqueFileName(const QUrl &url)
553{
554 QUrl cleanUrl = url;
555 cleanUrl.setPassword(password: QString());
556 cleanUrl.setFragment(fragment: QString());
557
558 const QByteArray hash = QCryptographicHash::hash(data: cleanUrl.toEncoded(), method: QCryptographicHash::Sha1);
559 // convert sha1 to base36 form and return first 8 bytes for use as string
560 const QByteArray id = QByteArray::number(*(qlonglong*)hash.data(), base: 36).left(len: 8);
561 // generates <one-char subdir>/<8-char filename.d>
562 uint code = (uint)id.at(i: id.size()-1) % 16;
563 QString pathFragment = QString::number(code, base: 16) + u'/' + QLatin1StringView(id) + CACHE_POSTFIX;
564
565 return pathFragment;
566}
567
568/*!
569 Generates fully qualified path of cached resource from a URL.
570 */
571QString QNetworkDiskCachePrivate::cacheFileName(const QUrl &url) const
572{
573 if (!url.isValid())
574 return QString();
575
576 QString fullpath = dataDirectory + uniqueFileName(url);
577 return fullpath;
578}
579
580/*!
581 We compress small text and JavaScript files.
582 */
583bool QCacheItem::canCompress() const
584{
585 bool sizeOk = false;
586 bool typeOk = false;
587 const auto headers = metaData.rawHeaders();
588 for (const auto &header : headers) {
589 if (header.first.compare(a: "content-length", cs: Qt::CaseInsensitive) == 0) {
590 qint64 size = header.second.toLongLong();
591 if (size > MAX_COMPRESSION_SIZE)
592 return false;
593 else
594 sizeOk = true;
595 }
596
597 if (header.first.compare(a: "content-type", cs: Qt::CaseInsensitive) == 0) {
598 QByteArray type = header.second;
599 if (type.startsWith(bv: "text/")
600 || (type.startsWith(bv: "application/")
601 && (type.endsWith(bv: "javascript") || type.endsWith(bv: "ecmascript"))))
602 typeOk = true;
603 else
604 return false;
605 }
606 if (sizeOk && typeOk)
607 return true;
608 }
609 return false;
610}
611
612enum
613{
614 CacheMagic = 0xe8,
615 CurrentCacheVersion = CACHE_VERSION
616};
617
618void QCacheItem::writeHeader(QFileDevice *device) const
619{
620 QDataStream out(device);
621
622 out << qint32(CacheMagic);
623 out << qint32(CurrentCacheVersion);
624 out << static_cast<qint32>(out.version());
625 out << metaData;
626 bool compressed = canCompress();
627 out << compressed;
628}
629
630void QCacheItem::writeCompressedData(QFileDevice *device) const
631{
632 QDataStream out(device);
633
634 out << qCompress(data: data.data());
635}
636
637/*!
638 Returns \c false if the file is a cache file,
639 but is an older version and should be removed otherwise true.
640 */
641bool QCacheItem::read(QFileDevice *device, bool readData)
642{
643 reset();
644
645 QDataStream in(device);
646
647 qint32 marker;
648 qint32 v;
649 in >> marker;
650 in >> v;
651 if (marker != CacheMagic)
652 return true;
653
654 // If the cache magic is correct, but the version is not we should remove it
655 if (v != CurrentCacheVersion)
656 return false;
657
658 qint32 streamVersion;
659 in >> streamVersion;
660 // Default stream version is also the highest we can handle
661 if (streamVersion > in.version())
662 return false;
663 in.setVersion(streamVersion);
664
665 bool compressed;
666 QByteArray dataBA;
667 in >> metaData;
668 in >> compressed;
669 if (readData && compressed) {
670 in >> dataBA;
671 data.setData(qUncompress(data: dataBA));
672 data.open(openMode: QBuffer::ReadOnly);
673 }
674
675 // quick and dirty check if metadata's URL field and the file's name are in synch
676 QString expectedFilename = QNetworkDiskCachePrivate::uniqueFileName(url: metaData.url());
677 if (!device->fileName().endsWith(s: expectedFilename))
678 return false;
679
680 return metaData.isValid() && !metaData.rawHeaders().isEmpty();
681}
682
683QT_END_NAMESPACE
684
685#include "moc_qnetworkdiskcache.cpp"
686

source code of qtbase/src/network/access/qnetworkdiskcache.cpp