1/*
2 SPDX-FileCopyrightText: 2016 Dan Leinir Turthra Jensen <admin@leinir.dk>
3
4 SPDX-License-Identifier: LGPL-2.1-or-later
5*/
6
7#include "httpworker.h"
8
9#include "knewstuff_version.h"
10#include "knewstuffcore_debug.h"
11
12#include <QCoreApplication>
13#include <QFile>
14#include <QMutex>
15#include <QMutexLocker>
16#include <QNetworkAccessManager>
17#include <QNetworkDiskCache>
18#include <QNetworkRequest>
19#include <QStandardPaths>
20#include <QStorageInfo>
21#include <QThread>
22
23namespace std
24{
25template<>
26struct default_delete<QNetworkReply> {
27 void operator()(QNetworkReply *ptr) const
28 {
29 ptr->abort();
30 ptr->deleteLater();
31 }
32};
33} // namespace std
34
35class HTTPWorkerNAM
36{
37public:
38 HTTPWorkerNAM()
39 {
40 QMutexLocker locker(&mutex);
41 const QString cacheLocation = QStandardPaths::writableLocation(type: QStandardPaths::CacheLocation) + QStringLiteral("/knewstuff");
42 cache.setCacheDirectory(cacheLocation);
43 QStorageInfo storageInfo(cacheLocation);
44 cache.setMaximumCacheSize(qMin(a: 50 * 1024 * 1024, b: (int)(storageInfo.bytesTotal() / 1000)));
45 nam.setCache(&cache);
46 }
47 QNetworkAccessManager nam;
48 QMutex mutex;
49
50 QNetworkReply *get(const QNetworkRequest &request)
51 {
52 QMutexLocker locker(&mutex);
53 return nam.get(request);
54 }
55
56 QNetworkDiskCache cache;
57};
58
59Q_GLOBAL_STATIC(HTTPWorkerNAM, s_httpWorkerNAM)
60
61using namespace KNSCore;
62
63class KNSCore::HTTPWorkerPrivate
64{
65public:
66 HTTPWorkerPrivate()
67 : jobType(HTTPWorker::GetJob)
68 , reply(nullptr)
69 {
70 }
71 HTTPWorker::JobType jobType;
72 QUrl source;
73 QUrl destination;
74 std::unique_ptr<QNetworkReply> reply;
75 QUrl redirectUrl;
76
77 QFile dataFile;
78};
79
80HTTPWorker::HTTPWorker(const QUrl &url, JobType jobType, QObject *parent)
81 : QObject(parent)
82 , d(new HTTPWorkerPrivate)
83{
84 d->jobType = jobType;
85 d->source = url;
86}
87
88HTTPWorker::HTTPWorker(const QUrl &source, const QUrl &destination, KNSCore::HTTPWorker::JobType jobType, QObject *parent)
89 : QObject(parent)
90 , d(new HTTPWorkerPrivate)
91{
92 d->jobType = jobType;
93 d->source = source;
94 d->destination = destination;
95}
96
97HTTPWorker::~HTTPWorker() = default;
98
99void HTTPWorker::setUrl(const QUrl &url)
100{
101 d->source = url;
102}
103
104static void addUserAgent(QNetworkRequest &request)
105{
106 QString agentHeader = QStringLiteral("KNewStuff/%1").arg(a: QLatin1String(KNEWSTUFF_VERSION_STRING));
107 if (QCoreApplication::instance()) {
108 agentHeader += QStringLiteral("-%1/%2").arg(args: QCoreApplication::instance()->applicationName(), args: QCoreApplication::instance()->applicationVersion());
109 }
110 request.setHeader(header: QNetworkRequest::UserAgentHeader, value: agentHeader);
111 // If the remote supports HTTP/2, then we should definitely be using that
112 request.setAttribute(code: QNetworkRequest::Http2AllowedAttribute, value: true);
113
114 // Assume that no cache expiration time will be longer than a week, but otherwise prefer the cache
115 // This is mildly hacky, but if we don't do this, we end up with infinite cache expirations in some
116 // cases, which of course isn't really acceptable... See ed62ee20 for a situation where that happened.
117 QNetworkCacheMetaData cacheMeta{s_httpWorkerNAM->cache.metaData(url: request.url())};
118 if (cacheMeta.isValid()) {
119 const QDateTime nextWeek{QDateTime::currentDateTime().addDays(days: 7)};
120 if (cacheMeta.expirationDate().isValid() && cacheMeta.expirationDate() < nextWeek) {
121 request.setAttribute(code: QNetworkRequest::CacheLoadControlAttribute, value: QNetworkRequest::PreferCache);
122 }
123 }
124}
125
126void HTTPWorker::startRequest()
127{
128 if (d->reply) {
129 // only run one request at a time...
130 return;
131 }
132
133 QNetworkRequest request(d->source);
134 addUserAgent(request);
135 d->reply.reset(p: s_httpWorkerNAM->get(request));
136 connect(sender: d->reply.get(), signal: &QNetworkReply::readyRead, context: this, slot: &HTTPWorker::handleReadyRead);
137 connect(sender: d->reply.get(), signal: &QNetworkReply::finished, context: this, slot: &HTTPWorker::handleFinished);
138 if (d->jobType == DownloadJob) {
139 d->dataFile.setFileName(d->destination.toLocalFile());
140 connect(sender: this, signal: &HTTPWorker::data, context: this, slot: &HTTPWorker::handleData);
141 }
142}
143
144void HTTPWorker::handleReadyRead()
145{
146 QMutexLocker locker(&s_httpWorkerNAM->mutex);
147 if (d->reply->attribute(code: QNetworkRequest::RedirectionTargetAttribute).isNull()) {
148 do {
149 Q_EMIT data(data: d->reply->read(maxlen: 32768));
150 } while (!d->reply->atEnd());
151 }
152}
153
154void HTTPWorker::handleFinished()
155{
156 qCDebug(KNEWSTUFFCORE) << Q_FUNC_INFO << d->reply->url();
157 if (d->reply->error() != QNetworkReply::NoError) {
158 qCWarning(KNEWSTUFFCORE) << d->reply->errorString();
159 if (d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt() > 100) {
160 // In this case, we're being asked to wait a bit...
161 Q_EMIT httpError(status: d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt(), rawHeaders: d->reply->rawHeaderPairs());
162 }
163 Q_EMIT error(error: d->reply->errorString());
164 }
165
166 // Check if the data was obtained from cache or not
167 QString fromCache = d->reply->attribute(code: QNetworkRequest::SourceIsFromCacheAttribute).toBool() ? QStringLiteral("(cached)") : QStringLiteral("(NOT cached)");
168
169 // Handle redirections
170 const QUrl possibleRedirectUrl = d->reply->attribute(code: QNetworkRequest::RedirectionTargetAttribute).toUrl();
171 if (!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != d->redirectUrl) {
172 d->redirectUrl = d->reply->url().resolved(relative: possibleRedirectUrl);
173 if (d->redirectUrl.scheme().startsWith(s: QLatin1String("http"))) {
174 qCDebug(KNEWSTUFFCORE) << d->reply->url().toDisplayString() << "was redirected to" << d->redirectUrl.toDisplayString() << fromCache
175 << d->reply->attribute(code: QNetworkRequest::HttpStatusCodeAttribute).toInt();
176 QNetworkRequest request(d->redirectUrl);
177 addUserAgent(request);
178 d->reply.reset(p: s_httpWorkerNAM->get(request));
179 connect(sender: d->reply.get(), signal: &QNetworkReply::readyRead, context: this, slot: &HTTPWorker::handleReadyRead);
180 connect(sender: d->reply.get(), signal: &QNetworkReply::finished, context: this, slot: &HTTPWorker::handleFinished);
181 return;
182 } else {
183 qCWarning(KNEWSTUFFCORE) << "Redirection to" << d->redirectUrl.toDisplayString() << "forbidden.";
184 }
185 } else {
186 qCDebug(KNEWSTUFFCORE) << "Data for" << d->reply->url().toDisplayString() << "was fetched" << fromCache;
187 }
188
189 if (d->dataFile.isOpen()) {
190 d->dataFile.close();
191 }
192
193 d->redirectUrl.clear();
194 Q_EMIT completed();
195}
196
197void HTTPWorker::handleData(const QByteArray &data)
198{
199 // It turns out that opening a file and then leaving it hanging without writing to it immediately will, at times
200 // leave you with a file that suddenly (seemingly magically) no longer exists. Thanks for that.
201 if (!d->dataFile.isOpen()) {
202 if (d->dataFile.open(flags: QIODevice::WriteOnly)) {
203 qCDebug(KNEWSTUFFCORE) << "Opened file" << d->dataFile.fileName() << "for writing.";
204 } else {
205 qCWarning(KNEWSTUFFCORE) << "Failed to open file for writing!";
206 Q_EMIT error(QStringLiteral("Failed to open file %1 for writing!").arg(a: d->destination.toLocalFile()));
207 }
208 }
209 qCDebug(KNEWSTUFFCORE) << "Writing" << data.length() << "bytes of data to" << d->dataFile.fileName();
210 quint64 written = d->dataFile.write(data);
211 if (d->dataFile.error()) {
212 qCDebug(KNEWSTUFFCORE) << "File has error" << d->dataFile.errorString();
213 }
214 qCDebug(KNEWSTUFFCORE) << "Wrote" << written << "bytes. File is now size" << d->dataFile.size();
215}
216
217#include "moc_httpworker.cpp"
218

source code of knewstuff/src/core/jobs/httpworker.cpp