File indexing completed on 2024-05-19 05:05:45
0001 /*************************************************************************** 0002 * SPDX-License-Identifier: GPL-2.0-or-later 0003 * * 0004 * SPDX-FileCopyrightText: 2004-2020 Thomas Fischer <fischer@unix-ag.uni-kl.de> 0005 * * 0006 * This program is free software; you can redistribute it and/or modify * 0007 * it under the terms of the GNU General Public License as published by * 0008 * the Free Software Foundation; either version 2 of the License, or * 0009 * (at your option) any later version. * 0010 * * 0011 * This program is distributed in the hope that it will be useful, * 0012 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 0013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 0014 * GNU General Public License for more details. * 0015 * * 0016 * You should have received a copy of the GNU General Public License * 0017 * along with this program; if not, see <https://www.gnu.org/licenses/>. * 0018 ***************************************************************************/ 0019 0020 #include "internalnetworkaccessmanager.h" 0021 0022 #include <ctime> 0023 0024 #include <QStringList> 0025 #include <QRegularExpression> 0026 #include <QNetworkAccessManager> 0027 #include <QNetworkCookieJar> 0028 #include <QNetworkCookie> 0029 #include <QNetworkProxy> 0030 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) 0031 #include <QNetworkProxyFactory> 0032 #endif // QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) 0033 #include <QNetworkRequest> 0034 #include <QNetworkReply> 0035 #include <QtGlobal> 0036 #include <QCoreApplication> 0037 #include <QTimer> 0038 #include <QUrl> 0039 #include <QUrlQuery> 0040 #if QT_VERSION >= 0x050a00 0041 #include <QRandomGenerator> 0042 #endif // QT_VERSION 0043 0044 #ifdef HAVE_KF 0045 #include <KProtocolManager> 0046 #endif // HAVE_KF 0047 0048 #include "logging_networking.h" 0049 0050 #if QT_VERSION >= 0x050a00 0051 #define randomGeneratorGlobalBounded(min,max) QRandomGenerator::global()->bounded((min),(max)) 0052 #else // QT_VERSION 0053 #define randomGeneratorGlobalBounded(min,max) ((min)+(qrand()%((max)-(min)+1))) 0054 #endif // QT_VERSION 0055 0056 /** 0057 * @author Thomas Fischer <fischer@unix-ag.uni-kl.de> 0058 */ 0059 class InternalNetworkAccessManager::HTTPEquivCookieJar: public QNetworkCookieJar 0060 { 0061 Q_OBJECT 0062 0063 public: 0064 void mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url) { 0065 static const QRegularExpression cookieContent(QStringLiteral("^([^\"=; ]+)=([^\"=; ]+).*\\bpath=([^\"=; ]+)"), QRegularExpression::CaseInsensitiveOption); 0066 int p1 = -1; 0067 QRegularExpressionMatch cookieContentRegExpMatch; 0068 if ((p1 = htmlCode.toLower().indexOf(QStringLiteral("http-equiv=\"set-cookie\""), 0, Qt::CaseInsensitive)) >= 5 0069 && (p1 = htmlCode.lastIndexOf(QStringLiteral("<meta"), p1, Qt::CaseInsensitive)) >= 0 0070 && (p1 = htmlCode.indexOf(QStringLiteral("content=\""), p1, Qt::CaseInsensitive)) >= 0 0071 && (cookieContentRegExpMatch = cookieContent.match(htmlCode.mid(p1 + 9, 512))).hasMatch()) { 0072 const QString key = cookieContentRegExpMatch.captured(1); 0073 const QString value = cookieContentRegExpMatch.captured(2); 0074 QList<QNetworkCookie> cookies = cookiesForUrl(url); 0075 cookies.append(QNetworkCookie(key.toLatin1(), value.toLatin1())); 0076 setCookiesFromUrl(cookies, url); 0077 } 0078 } 0079 0080 HTTPEquivCookieJar(QObject *parent = nullptr) 0081 : QNetworkCookieJar(parent) { 0082 /// nothing 0083 } 0084 }; 0085 0086 0087 QString InternalNetworkAccessManager::userAgentString; 0088 0089 InternalNetworkAccessManager::InternalNetworkAccessManager(QObject *parent) 0090 : QNetworkAccessManager(parent) 0091 { 0092 cookieJar = new HTTPEquivCookieJar(this); 0093 #if QT_VERSION < 0x050a00 0094 qsrand(static_cast<int>(QDateTime::currentDateTime().toMSecsSinceEpoch() % 0x7fffffffl)); 0095 #endif // QT_VERSION 0096 } 0097 0098 0099 void InternalNetworkAccessManager::mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url) 0100 { 0101 Q_ASSERT_X(cookieJar != nullptr, "void InternalNetworkAccessManager::mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url)", "cookieJar is invalid"); 0102 cookieJar->mergeHtmlHeadCookies(htmlCode, url); 0103 setCookieJar(cookieJar); 0104 } 0105 0106 InternalNetworkAccessManager &InternalNetworkAccessManager::instance() 0107 { 0108 static InternalNetworkAccessManager self; 0109 return self; 0110 } 0111 0112 QNetworkReply *InternalNetworkAccessManager::get(QNetworkRequest &request, const QUrl &oldUrl) 0113 { 0114 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0115 #ifdef HAVE_KF 0116 /// Query the KDE subsystem if a proxy has to be used 0117 /// for the host of a given URL 0118 QString proxyHostName = KProtocolManager::proxyForUrl(request.url()); 0119 if (!proxyHostName.isEmpty() && proxyHostName != QStringLiteral("DIRECT")) { 0120 /// Extract both hostname and port number for proxy 0121 proxyHostName = proxyHostName.mid(proxyHostName.indexOf(QStringLiteral("://")) + 3); 0122 #if QT_VERSION >= 0x050e00 0123 QStringList proxyComponents = proxyHostName.split(QStringLiteral(":"), Qt::SkipEmptyParts); 0124 #else // QT_VERSION < 0x050e00 0125 QStringList proxyComponents = proxyHostName.split(QStringLiteral(":"), QString::SkipEmptyParts); 0126 #endif // QT_VERSION >= 0x050e00 0127 if (proxyComponents.length() == 1) { 0128 /// Proxy configuration is missing a port number, 0129 /// using 8080 as default 0130 proxyComponents << QStringLiteral("8080"); 0131 } 0132 if (proxyComponents.length() == 2) { 0133 /// Set proxy to Qt's NetworkAccessManager 0134 setProxy(QNetworkProxy(QNetworkProxy::HttpProxy, proxyComponents[0], proxyComponents[1].toInt())); 0135 } 0136 } else { 0137 /// No proxy to be used, clear previous settings 0138 setProxy(QNetworkProxy()); 0139 } 0140 #else // HAVE_KF 0141 setProxy(QNetworkProxy()); 0142 #endif // HAVE_KF 0143 #else // QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) 0144 const auto networkProxyList = QNetworkProxyFactory::proxyForQuery(QNetworkProxyQuery(request.url())); 0145 if (networkProxyList.length() < 1 || networkProxyList.first().type() == QNetworkProxy::NoProxy) 0146 setProxy(QNetworkProxy()); 0147 else 0148 setProxy(networkProxyList.first()); 0149 #endif // QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0150 0151 if (!request.hasRawHeader(QByteArray("Accept"))) 0152 request.setRawHeader(QByteArray("Accept"), QByteArray("text/*, */*;q=0.7")); 0153 request.setRawHeader(QByteArray("Accept-Charset"), QByteArray("utf-8, us-ascii, ISO-8859-1;q=0.7, ISO-8859-15;q=0.7, windows-1252;q=0.3")); 0154 request.setRawHeader(QByteArray("Accept-Language"), QByteArray("en-US, en;q=0.9")); 0155 /// Set 'Referer' and 'Origin' to match the request URL's domain, i.e. URL with empty path 0156 QUrl domainUrl = request.url(); 0157 domainUrl.setPath(QString()); 0158 const QByteArray domain = removeApiKey(domainUrl).toDisplayString().toLatin1(); 0159 request.setRawHeader(QByteArray("Referer"), domain); 0160 request.setRawHeader(QByteArray("Origin"), domain); 0161 request.setRawHeader(QByteArray("User-Agent"), userAgent().toLatin1()); 0162 if (oldUrl.isValid()) 0163 request.setRawHeader(QByteArray("Referer"), removeApiKey(oldUrl).toDisplayString().toLatin1()); 0164 QNetworkReply *reply = QNetworkAccessManager::get(request); 0165 0166 /// Log SSL errors 0167 connect(reply, &QNetworkReply::sslErrors, this, &InternalNetworkAccessManager::logSslErrors); 0168 0169 return reply; 0170 } 0171 0172 QNetworkReply *InternalNetworkAccessManager::get(QNetworkRequest &request, const QNetworkReply *oldReply) 0173 { 0174 return get(request, oldReply == nullptr ? QUrl() : oldReply->url()); 0175 } 0176 0177 QString InternalNetworkAccessManager::userAgent() 0178 { 0179 /// Various browser strings to "disguise" origin 0180 if (userAgentString.isEmpty()) { 0181 if (randomGeneratorGlobalBounded(0, 1) == 0) { 0182 /// Fake Chrome user agent string 0183 static const QString chromeTemplate{QStringLiteral("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/%1.%2 (KHTML, like Gecko) Chrome/%3.%4.%5.%6 Safari/%1.%2")}; 0184 const auto appleWebKitVersionMajor = randomGeneratorGlobalBounded(537, 856); 0185 const auto appleWebKitVersionMinor = randomGeneratorGlobalBounded(3, 53); 0186 const auto chromeVersionMajor = randomGeneratorGlobalBounded(77, 85); 0187 const auto chromeVersionMinor = randomGeneratorGlobalBounded(0, 4); 0188 const auto chromeVersionBuild = randomGeneratorGlobalBounded(3793, 8973); 0189 const auto chromeVersionPatch = randomGeneratorGlobalBounded(53, 673); 0190 userAgentString = chromeTemplate.arg(appleWebKitVersionMajor).arg(appleWebKitVersionMinor).arg(chromeVersionMajor).arg(chromeVersionMinor).arg(chromeVersionBuild).arg(chromeVersionPatch); 0191 } else { 0192 /// Fake Firefox user agent string 0193 static const QString mozillaTemplate{QStringLiteral("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:%1.%2) Gecko/20100101 Firefox/%1.%2")}; 0194 const auto firefoxVersionMajor = randomGeneratorGlobalBounded(69, 74); 0195 const auto firefoxVersionMinor = randomGeneratorGlobalBounded(0, 2); 0196 userAgentString = mozillaTemplate.arg(firefoxVersionMajor).arg(firefoxVersionMinor); 0197 } 0198 } 0199 return userAgentString; 0200 } 0201 0202 void InternalNetworkAccessManager::setNetworkReplyTimeout(QNetworkReply *reply, int timeOutSec) 0203 { 0204 QTimer *timer = new QTimer(reply); 0205 connect(timer, &QTimer::timeout, this, &InternalNetworkAccessManager::networkReplyTimeout); 0206 m_mapTimerToReply.insert(timer, reply); 0207 timer->start(timeOutSec * 1000); 0208 connect(reply, &QNetworkReply::finished, this, &InternalNetworkAccessManager::networkReplyFinished); 0209 } 0210 0211 QString InternalNetworkAccessManager::reverseObfuscate(const QByteArray &a) { 0212 if (a.length() % 2 != 0 || a.length() == 0) return QString(); 0213 QString result; 0214 result.reserve(a.length() / 2); 0215 for (int p = a.length() - 1; p >= 0; p -= 2) { 0216 const QChar c = QLatin1Char(a.at(p) ^ a.at(p - 1)); 0217 result.append(c); 0218 } 0219 return result; 0220 } 0221 0222 QUrl InternalNetworkAccessManager::removeApiKey(QUrl url) 0223 { 0224 QUrlQuery urlQuery(url); 0225 urlQuery.removeQueryItem(QStringLiteral("apikey")); 0226 urlQuery.removeQueryItem(QStringLiteral("api_key")); 0227 url.setQuery(urlQuery); 0228 return url; 0229 } 0230 0231 QString InternalNetworkAccessManager::removeApiKey(const QString &text) 0232 { 0233 static const QRegularExpression apiKeyRegExp(QStringLiteral("\\bapi_?key=[^\"&? ]")); 0234 return QString(text).remove(apiKeyRegExp); 0235 } 0236 0237 void InternalNetworkAccessManager::networkReplyTimeout() 0238 { 0239 QTimer *timer = static_cast<QTimer *>(sender()); 0240 timer->stop(); 0241 QNetworkReply *reply = m_mapTimerToReply[timer]; 0242 if (reply != nullptr) { 0243 qCWarning(LOG_KBIBTEX_NETWORKING) << "Timeout on reply to " << removeApiKey(reply->url()).toDisplayString(); 0244 reply->close(); 0245 m_mapTimerToReply.remove(timer); 0246 } 0247 } 0248 void InternalNetworkAccessManager::networkReplyFinished() 0249 { 0250 QNetworkReply *reply = static_cast<QNetworkReply *>(sender()); 0251 QTimer *timer = m_mapTimerToReply.key(reply, nullptr); 0252 if (timer != nullptr) { 0253 disconnect(timer, &QTimer::timeout, this, &InternalNetworkAccessManager::networkReplyTimeout); 0254 timer->stop(); 0255 m_mapTimerToReply.remove(timer); 0256 } 0257 } 0258 0259 void InternalNetworkAccessManager::logSslErrors(const QList<QSslError> &errors) 0260 { 0261 QNetworkReply *reply = static_cast<QNetworkReply *>(sender()); 0262 qCWarning(LOG_KBIBTEX_NETWORKING) << QStringLiteral("Got the following SSL errors when querying the following URL: ") << removeApiKey(reply->url()).toDisplayString(); 0263 for (const QSslError &error : errors) 0264 qCWarning(LOG_KBIBTEX_NETWORKING) << QStringLiteral(" * ") + error.errorString() << "; Code: " << static_cast<int>(error.error()); 0265 } 0266 0267 #include "internalnetworkaccessmanager.moc"