1/*
2 SPDX-FileCopyrightText: 2016 Dan Leinir Turthra Jensen <admin@leinir.dk>
3
4 SPDX-License-Identifier: LGPL-2.1-or-later
5*/
6
7#include "httpworker.h"
8
9#include "knewstuff_version.h"
10#include "knewstuffcore_debug.h"
11
12#include <QCoreApplication>
13#include <QFile>
14#include <QMutex>
15#include <QMutexLocker>
16#include <QNetworkAccessManager>
17#include <QNetworkDiskCache>
18#include <QNetworkRequest>
19#include <QStandardPaths>
20#include <QStorageInfo>
21#include <QThread>
22
23class HTTPWorkerNAM
24{
25public:
26 HTTPWorkerNAM()
27 {
28 QMutexLocker locker(&mutex);
29 const QString cacheLocation = QStandardPaths::writableLocation(type: QStandardPaths::CacheLocation) + QStringLiteral("/knewstuff");
30 cache.setCacheDirectory(cacheLocation);
31 QStorageInfo storageInfo(cacheLocation);
32 cache.setMaximumCacheSize(qMin(a: 50 * 1024 * 1024, b: (int)(storageInfo.bytesTotal() / 1000)));
33 nam.setCache(&cache);
34 }
35 QNetworkAccessManager nam;
36 QMutex mutex;
37
38 QNetworkReply *get(const QNetworkRequest &request)
39 {
40 QMutexLocker locker(&mutex);
41 return nam.get(request);
42 }
43
44 QNetworkDiskCache cache;
45};
46
47Q_GLOBAL_STATIC(HTTPWorkerNAM, s_httpWorkerNAM)
48
49using namespace KNSCore;
50
51class KNSCore::HTTPWorkerPrivate
52{
53public:
54 HTTPWorkerPrivate()
55 : jobType(HTTPWorker::GetJob)
56 , reply(nullptr)
57 {
58 }
59 HTTPWorker::JobType jobType;
60 QUrl source;
61 QUrl destination;
62 QNetworkReply *reply;
63 QUrl redirectUrl;
64
65 QFile dataFile;
66};
67
68HTTPWorker::HTTPWorker(const QUrl &url, JobType jobType, QObject *parent)
69 : QObject(parent)
70 , d(new HTTPWorkerPrivate)
71{
72 d->jobType = jobType;
73 d->source = url;
74}
75
76HTTPWorker::HTTPWorker(const QUrl &source, const QUrl &destination, KNSCore::HTTPWorker::JobType jobType, QObject *parent)
77 : QObject(parent)
78 , d(new HTTPWorkerPrivate)
79{
80 d->jobType = jobType;
81 d->source = source;
82 d->destination = destination;
83}
84
85HTTPWorker::~HTTPWorker() = default;
86
87void HTTPWorker::setUrl(const QUrl &url)
88{
89 d->source = url;
90}
91
92static void addUserAgent(QNetworkRequest &request)
93{
94 QString agentHeader = QStringLiteral("KNewStuff/%1").arg(a: QLatin1String(KNEWSTUFF_VERSION_STRING));
95 if (QCoreApplication::instance()) {
96 agentHeader += QStringLiteral("-%1/%2").arg(args: QCoreApplication::instance()->applicationName(), args: QCoreApplication::instance()->applicationVersion());
97 }
98 request.setHeader(header: QNetworkRequest::UserAgentHeader, value: agentHeader);
99 // If the remote supports HTTP/2, then we should definitely be using that
100 request.setAttribute(code: QNetworkRequest::Http2AllowedAttribute, value: true);
101
102 // Assume that no cache expiration time will be longer than a week, but otherwise prefer the cache
103 // This is mildly hacky, but if we don't do this, we end up with infinite cache expirations in some
104 // cases, which of course isn't really acceptable... See ed62ee20 for a situation where that happened.
105 QNetworkCacheMetaData cacheMeta{s_httpWorkerNAM->cache.metaData(url: request.url())};
106 if (cacheMeta.isValid()) {
107 const QDateTime nextWeek{QDateTime::currentDateTime().addDays(days: 7)};
108 if (cacheMeta.expirationDate().isValid() && cacheMeta.expirationDate() < nextWeek) {
109 request.setAttribute(code: QNetworkRequest::CacheLoadControlAttribute, value: QNetworkRequest::PreferCache);
110 }
111 }
112}
113
114void HTTPWorker::startRequest()
115{
116 if (d->reply) {
117 // only run one request at a time...
118 return;
119 }
120
121 QNetworkRequest request(d->source);
122 addUserAgent(request);
123 d->reply = s_httpWorkerNAM->get(request);
124 connect(sender: d->reply, signal: &QNetworkReply::readyRead, context: this, slot: &HTTPWorker::handleReadyRead);
125 connect(sender: d->reply, signal: &QNetworkReply::finished, context: this, slot: &HTTPWorker::handleFinished);
126 if (d->jobType == DownloadJob) {
127 d->dataFile.setFileName(d->destination.toLocalFile());
128 connect(sender: this, signal: &HTTPWorker::data, context: this, slot: &HTTPWorker::handleData);
129 }
130}
131
132void HTTPWorker::handleReadyRead()
133{
134 QMutexLocker locker(&s_httpWorkerNAM->mutex);
135 if (d->reply->attribute(code: QNetworkRequest::RedirectionTargetAttribute).isNull()) {
136 do {
137 Q_EMIT data(data: d->reply->read(maxlen: 32768));
138 } while (!d->reply->atEnd());
139 }
140}
141
142void HTTPWorker::handleFinished()
143{
144 qCDebug(KNEWSTUFFCORE) << Q_FUNC_INFO << d->reply->url();
145 if (d->reply->error() != QNetworkReply::NoError) {
146 qCWarning(KNEWSTUFFCORE) << d->reply->errorString();
147 if (d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt() > 100) {
148 // In this case, we're being asked to wait a bit...
149 Q_EMIT httpError(status: d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt(), rawHeaders: d->reply->rawHeaderPairs());
150 }
151 Q_EMIT error(error: d->reply->errorString());
152 }
153
154 // Check if the data was obtained from cache or not
155 QString fromCache = d->reply->attribute(code: QNetworkRequest::SourceIsFromCacheAttribute).toBool() ? QStringLiteral("(cached)") : QStringLiteral("(NOT cached)");
156
157 // Handle redirections
158 const QUrl possibleRedirectUrl = d->reply->attribute(code: QNetworkRequest::RedirectionTargetAttribute).toUrl();
159 if (!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != d->redirectUrl) {
160 d->redirectUrl = d->reply->url().resolved(relative: possibleRedirectUrl);
161 if (d->redirectUrl.scheme().startsWith(s: QLatin1String("http"))) {
162 qCDebug(KNEWSTUFFCORE) << d->reply->url().toDisplayString() << "was redirected to" << d->redirectUrl.toDisplayString() << fromCache
163 << d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt();
164 d->reply->deleteLater();
165 QNetworkRequest request(d->redirectUrl);
166 addUserAgent(request);
167 d->reply = s_httpWorkerNAM->get(request);
168 connect(sender: d->reply, signal: &QNetworkReply::readyRead, context: this, slot: &HTTPWorker::handleReadyRead);
169 connect(sender: d->reply, signal: &QNetworkReply::finished, context: this, slot: &HTTPWorker::handleFinished);
170 return;
171 } else {
172 qCWarning(KNEWSTUFFCORE) << "Redirection to" << d->redirectUrl.toDisplayString() << "forbidden.";
173 }
174 } else {
175 qCDebug(KNEWSTUFFCORE) << "Data for" << d->reply->url().toDisplayString() << "was fetched" << fromCache;
176 }
177
178 if (d->dataFile.isOpen()) {
179 d->dataFile.close();
180 }
181
182 d->redirectUrl.clear();
183 Q_EMIT completed();
184}
185
186void HTTPWorker::handleData(const QByteArray &data)
187{
188 // It turns out that opening a file and then leaving it hanging without writing to it immediately will, at times
189 // leave you with a file that suddenly (seemingly magically) no longer exists. Thanks for that.
190 if (!d->dataFile.isOpen()) {
191 if (d->dataFile.open(flags: QIODevice::WriteOnly)) {
192 qCDebug(KNEWSTUFFCORE) << "Opened file" << d->dataFile.fileName() << "for writing.";
193 } else {
194 qCWarning(KNEWSTUFFCORE) << "Failed to open file for writing!";
195 Q_EMIT error(QStringLiteral("Failed to open file %1 for writing!").arg(a: d->destination.toLocalFile()));
196 }
197 }
198 qCDebug(KNEWSTUFFCORE) << "Writing" << data.length() << "bytes of data to" << d->dataFile.fileName();
199 quint64 written = d->dataFile.write(data);
200 if (d->dataFile.error()) {
201 qCDebug(KNEWSTUFFCORE) << "File has error" << d->dataFile.errorString();
202 }
203 qCDebug(KNEWSTUFFCORE) << "Wrote" << written << "bytes. File is now size" << d->dataFile.size();
204}
205
206#include "moc_httpworker.cpp"
207

source code of knewstuff/src/core/jobs/httpworker.cpp