File indexing completed on 2024-05-12 15:43:55
0001 /* 0002 SPDX-FileCopyrightText: 2016 Dan Leinir Turthra Jensen <admin@leinir.dk> 0003 0004 SPDX-License-Identifier: LGPL-2.1-or-later 0005 */ 0006 0007 #include "httpworker.h" 0008 0009 #include "knewstuff_version.h" 0010 #include "knewstuffcore_debug.h" 0011 0012 #include <QCoreApplication> 0013 #include <QFile> 0014 #include <QMutex> 0015 #include <QMutexLocker> 0016 #include <QNetworkAccessManager> 0017 #include <QNetworkDiskCache> 0018 #include <QNetworkRequest> 0019 #include <QStandardPaths> 0020 #include <QStorageInfo> 0021 #include <QThread> 0022 0023 class HTTPWorkerNAM 0024 { 0025 public: 0026 HTTPWorkerNAM() 0027 { 0028 QMutexLocker locker(&mutex); 0029 const QString cacheLocation = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/knewstuff"); 0030 cache.setCacheDirectory(cacheLocation); 0031 QStorageInfo storageInfo(cacheLocation); 0032 cache.setMaximumCacheSize(qMin(50 * 1024 * 1024, (int)(storageInfo.bytesTotal() / 1000))); 0033 nam.setCache(&cache); 0034 } 0035 QNetworkAccessManager nam; 0036 QMutex mutex; 0037 0038 QNetworkReply *get(const QNetworkRequest &request) 0039 { 0040 QMutexLocker locker(&mutex); 0041 return nam.get(request); 0042 } 0043 0044 QNetworkDiskCache cache; 0045 }; 0046 0047 Q_GLOBAL_STATIC(HTTPWorkerNAM, s_httpWorkerNAM) 0048 0049 using namespace KNSCore; 0050 0051 class KNSCore::HTTPWorkerPrivate 0052 { 0053 public: 0054 HTTPWorkerPrivate() 0055 : jobType(HTTPWorker::GetJob) 0056 , reply(nullptr) 0057 { 0058 } 0059 HTTPWorker::JobType jobType; 0060 QUrl source; 0061 QUrl destination; 0062 QNetworkReply *reply; 0063 QUrl redirectUrl; 0064 0065 QFile dataFile; 0066 }; 0067 0068 HTTPWorker::HTTPWorker(const QUrl &url, JobType jobType, QObject *parent) 0069 : QObject(parent) 0070 , d(new HTTPWorkerPrivate) 0071 { 0072 qCDebug(KNEWSTUFFCORE) << Q_FUNC_INFO; 0073 d->jobType = jobType; 0074 d->source = url; 0075 } 0076 0077 HTTPWorker::HTTPWorker(const QUrl &source, const QUrl &destination, KNSCore::HTTPWorker::JobType jobType, QObject *parent) 0078 : QObject(parent) 0079 , d(new HTTPWorkerPrivate) 0080 { 0081 qCDebug(KNEWSTUFFCORE) << Q_FUNC_INFO; 0082 d->jobType = jobType; 0083 d->source = source; 0084 d->destination = destination; 0085 } 0086 0087 HTTPWorker::~HTTPWorker() = default; 0088 0089 void HTTPWorker::setUrl(const QUrl &url) 0090 { 0091 d->source = url; 0092 } 0093 0094 static void addUserAgent(QNetworkRequest &request) 0095 { 0096 QString agentHeader = QStringLiteral("KNewStuff/%1").arg(QLatin1String(KNEWSTUFF_VERSION_STRING)); 0097 if (QCoreApplication::instance()) { 0098 agentHeader += QStringLiteral("-%1/%2").arg(QCoreApplication::instance()->applicationName(), QCoreApplication::instance()->applicationVersion()); 0099 } 0100 request.setHeader(QNetworkRequest::UserAgentHeader, agentHeader); 0101 // If the remote supports HTTP/2, then we should definitely be using that 0102 request.setAttribute(QNetworkRequest::Http2AllowedAttribute, true); 0103 0104 // Assume that no cache expiration time will be longer than a week, but otherwise prefer the cache 0105 // This is mildly hacky, but if we don't do this, we end up with infinite cache expirations in some 0106 // cases, which of course isn't really acceptable... See ed62ee20 for a situation where that happened. 0107 QNetworkCacheMetaData cacheMeta{s_httpWorkerNAM->cache.metaData(request.url())}; 0108 if (cacheMeta.isValid()) { 0109 const QDateTime nextWeek{QDateTime::currentDateTime().addDays(7)}; 0110 if (cacheMeta.expirationDate().isValid() && cacheMeta.expirationDate() < nextWeek) { 0111 request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); 0112 } 0113 } 0114 } 0115 0116 void HTTPWorker::startRequest() 0117 { 0118 if (d->reply) { 0119 // only run one request at a time... 0120 return; 0121 } 0122 0123 QNetworkRequest request(d->source); 0124 addUserAgent(request); 0125 d->reply = s_httpWorkerNAM->get(request); 0126 connect(d->reply, &QNetworkReply::readyRead, this, &HTTPWorker::handleReadyRead); 0127 connect(d->reply, &QNetworkReply::finished, this, &HTTPWorker::handleFinished); 0128 if (d->jobType == DownloadJob) { 0129 d->dataFile.setFileName(d->destination.toLocalFile()); 0130 connect(this, &HTTPWorker::data, this, &HTTPWorker::handleData); 0131 } 0132 } 0133 0134 void HTTPWorker::handleReadyRead() 0135 { 0136 QMutexLocker locker(&s_httpWorkerNAM->mutex); 0137 if (d->reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isNull()) { 0138 do { 0139 Q_EMIT data(d->reply->read(32768)); 0140 } while (!d->reply->atEnd()); 0141 } 0142 } 0143 0144 void HTTPWorker::handleFinished() 0145 { 0146 qCDebug(KNEWSTUFFCORE) << Q_FUNC_INFO << d->reply->url(); 0147 if (d->reply->error() != QNetworkReply::NoError) { 0148 qCWarning(KNEWSTUFFCORE) << d->reply->errorString(); 0149 if (d->reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() > 100) { 0150 // In this case, we're being asked to wait a bit... 0151 Q_EMIT httpError(d->reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(), d->reply->rawHeaderPairs()); 0152 } 0153 Q_EMIT error(d->reply->errorString()); 0154 } 0155 0156 // Check if the data was obtained from cache or not 0157 QString fromCache = d->reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool() ? QStringLiteral("(cached)") : QStringLiteral("(NOT cached)"); 0158 0159 // Handle redirections 0160 const QUrl possibleRedirectUrl = d->reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl(); 0161 if (!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != d->redirectUrl) { 0162 d->redirectUrl = d->reply->url().resolved(possibleRedirectUrl); 0163 if (d->redirectUrl.scheme().startsWith(QLatin1String("http"))) { 0164 qCDebug(KNEWSTUFFCORE) << d->reply->url().toDisplayString() << "was redirected to" << d->redirectUrl.toDisplayString() << fromCache 0165 << d->reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); 0166 d->reply->deleteLater(); 0167 QNetworkRequest request(d->redirectUrl); 0168 addUserAgent(request); 0169 d->reply = s_httpWorkerNAM->get(request); 0170 connect(d->reply, &QNetworkReply::readyRead, this, &HTTPWorker::handleReadyRead); 0171 connect(d->reply, &QNetworkReply::finished, this, &HTTPWorker::handleFinished); 0172 return; 0173 } else { 0174 qCWarning(KNEWSTUFFCORE) << "Redirection to" << d->redirectUrl.toDisplayString() << "forbidden."; 0175 } 0176 } else { 0177 qCDebug(KNEWSTUFFCORE) << "Data for" << d->reply->url().toDisplayString() << "was fetched" << fromCache; 0178 } 0179 0180 if (d->dataFile.isOpen()) { 0181 d->dataFile.close(); 0182 } 0183 0184 d->redirectUrl.clear(); 0185 Q_EMIT completed(); 0186 } 0187 0188 void HTTPWorker::handleData(const QByteArray &data) 0189 { 0190 // It turns out that opening a file and then leaving it hanging without writing to it immediately will, at times 0191 // leave you with a file that suddenly (seemingly magically) no longer exists. Thanks for that. 0192 if (!d->dataFile.isOpen()) { 0193 if (d->dataFile.open(QIODevice::WriteOnly)) { 0194 qCDebug(KNEWSTUFFCORE) << "Opened file" << d->dataFile.fileName() << "for writing."; 0195 } else { 0196 qCWarning(KNEWSTUFFCORE) << "Failed to open file for writing!"; 0197 Q_EMIT error(QStringLiteral("Failed to open file %1 for writing!").arg(d->destination.toLocalFile())); 0198 } 0199 } 0200 qCDebug(KNEWSTUFFCORE) << "Writing" << data.length() << "bytes of data to" << d->dataFile.fileName(); 0201 quint64 written = d->dataFile.write(data); 0202 if (d->dataFile.error()) { 0203 qCDebug(KNEWSTUFFCORE) << "File has error" << d->dataFile.errorString(); 0204 } 0205 qCDebug(KNEWSTUFFCORE) << "Wrote" << written << "bytes. File is now size" << d->dataFile.size(); 0206 } 0207 0208 #include "moc_httpworker.cpp"