File indexing completed on 2024-05-12 04:42:18
0001 /* 0002 SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org> 0003 0004 SPDX-License-Identifier: LGPL-2.0-or-later 0005 */ 0006 0007 #include "overpassquerymanager.h" 0008 #include "overpassquery.h" 0009 0010 #include <QDateTime> 0011 #include <QNetworkAccessManager> 0012 #include <QNetworkDiskCache> 0013 #include <QNetworkReply> 0014 #include <QStandardPaths> 0015 #include <QTimer> 0016 #include <QUrl> 0017 #include <QUrlQuery> 0018 0019 #include <chrono> 0020 #include <deque> 0021 0022 using namespace OSM; 0023 0024 namespace OSM { 0025 struct OverpassQueryTask { 0026 OverpassQuery *query = nullptr; 0027 QRectF bbox; 0028 bool forceReload = false; 0029 }; 0030 0031 struct OverpassQueryExecutor { 0032 QUrl endpoint; 0033 std::chrono::seconds cooldownTime = std::chrono::seconds(3); 0034 QDateTime nextSlot; 0035 std::unique_ptr<OverpassQueryTask> task; 0036 }; 0037 0038 class OverpassQueryManagerPrivate { 0039 public: 0040 void executeTasks(); 0041 void taskFinished(OverpassQueryExecutor *executor, QNetworkReply *reply); 0042 void checkQueryFinished(OverpassQuery *query) const; 0043 void cancelQuery(OverpassQuery *query); 0044 0045 OverpassQueryManager *q; 0046 QNetworkAccessManager *m_nam; 0047 QTimer *m_nextTaskTimer; 0048 std::vector<OverpassQueryExecutor> m_executors; 0049 std::deque<std::unique_ptr<OverpassQueryTask>> m_tasks; 0050 }; 0051 } 0052 0053 static const char* executor_configs[] = { 0054 "https://overpass-api.de/api/interpreter", 0055 "https://overpass.openstreetmap.fr/api/interpreter", 0056 // "https://1.overpass.kumi.systems/api/interpreter", 0057 // "https://2.overpass.kumi.systems/api/interpreter", 0058 // "https://3.overpass.kumi.systems/api/interpreter", 0059 // "https://4.overpass.kumi.systems/api/interpreter", 0060 }; 0061 0062 OverpassQueryManager::OverpassQueryManager(QObject *parent) 0063 : QObject(parent) 0064 , d(new OverpassQueryManagerPrivate) 0065 { 0066 d->q = this; 0067 d->m_nam = new QNetworkAccessManager(this); 0068 d->m_nam->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy); 0069 d->m_nam->setStrictTransportSecurityEnabled(true); 0070 d->m_nam->enableStrictTransportSecurityStore(true, QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.osm/hsts/")); 0071 0072 auto diskCache = new QNetworkDiskCache; 0073 diskCache->setCacheDirectory(QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.osm/overpass-cache/")); 0074 diskCache->setMaximumCacheSize(1'000'000'000); // 1GB 0075 d->m_nam->setCache(diskCache); 0076 0077 d->m_nextTaskTimer = new QTimer(this); 0078 d->m_nextTaskTimer->setSingleShot(true); 0079 connect(d->m_nextTaskTimer, &QTimer::timeout, this, [this]() { d->executeTasks(); }); 0080 0081 for (const auto &config : executor_configs) { 0082 OverpassQueryExecutor executor; 0083 executor.endpoint = QUrl(QString::fromUtf8(config)); 0084 d->m_executors.push_back(std::move(executor)); 0085 } 0086 } 0087 0088 OverpassQueryManager::~OverpassQueryManager() = default; 0089 0090 void OverpassQueryManager::execute(OverpassQuery *query) 0091 { 0092 // validate input 0093 if (query->query().isEmpty() || query->boundingBox().isNull() || !query->boundingBox().isValid() || query->tileSize().isNull() || !query->tileSize().isValid()) { 0094 query->m_error = OverpassQuery::QueryError; 0095 QTimer::singleShot(0, query, &OverpassQuery::finished); 0096 return; 0097 } 0098 0099 // generate tasks for the query 0100 const auto xTileCount = std::max<int>(1, query->boundingBox().width() / query->tileSize().width()); 0101 const auto yTileCount = std::max<int>(1, query->boundingBox().height() / query->tileSize().height()); 0102 const auto xTileSize = query->boundingBox().width() / xTileCount; 0103 const auto yTileSize = query->boundingBox().height() / yTileCount; 0104 qDebug() << "Creating" << xTileCount * yTileCount << "tasks with tile size" << xTileSize << "x" << yTileSize; 0105 for (auto x = 0; x < xTileCount; ++x) { 0106 for (auto y = 0; y < yTileCount; ++y) { 0107 auto task = std::make_unique<OverpassQueryTask>(); 0108 task->query = query; 0109 task->bbox = { query->boundingBox().x() + x * xTileSize, query->boundingBox().y() + y * yTileSize, xTileSize, yTileSize }; 0110 d->m_tasks.push_back(std::move(task)); 0111 } 0112 } 0113 0114 d->executeTasks(); 0115 } 0116 0117 void OverpassQueryManagerPrivate::executeTasks() 0118 { 0119 const auto now = QDateTime::currentDateTimeUtc(); 0120 std::chrono::seconds nextSlot = std::chrono::hours(1); 0121 0122 for (auto &executor : m_executors) { 0123 if (m_tasks.empty()) { // nothing to do 0124 return; 0125 } 0126 0127 if (executor.task) { // executor is busy already 0128 continue; 0129 } 0130 0131 if (executor.nextSlot > now) { // executor is still in rate limit cooldown 0132 nextSlot = std::min(std::chrono::seconds(now.secsTo(executor.nextSlot)), nextSlot); 0133 nextSlot += std::chrono::seconds(1); // for msec rounding errors that would other wise give us a busy loop 0134 if (m_tasks.front()->forceReload) { 0135 continue; 0136 } 0137 } 0138 0139 executor.task = std::move(m_tasks.front()); 0140 m_tasks.pop_front(); 0141 0142 // actually execute query 0143 auto url = executor.endpoint; 0144 QUrlQuery params; 0145 params.addQueryItem(QStringLiteral("data"), executor.task->query->query(executor.task->bbox)); 0146 url.setQuery(params); 0147 QNetworkRequest req(url); 0148 req.setAttribute(QNetworkRequest::CacheLoadControlAttribute, executor.task->forceReload ? QNetworkRequest::PreferNetwork : QNetworkRequest::AlwaysCache); 0149 auto reply = m_nam->get(req); 0150 // TODO enable stream parsing for XML replies by connecting to QNetworkReply::readyRead 0151 QObject::connect(reply, &QNetworkReply::finished, q, [this, &executor, reply]() { 0152 taskFinished(&executor, reply); 0153 reply->deleteLater(); 0154 }); 0155 } 0156 0157 m_nextTaskTimer->start(nextSlot); 0158 } 0159 0160 void OverpassQueryManagerPrivate::taskFinished(OverpassQueryExecutor *executor, QNetworkReply *reply) 0161 { 0162 auto query = executor->task->query; 0163 if (reply->error() == QNetworkReply::UnknownContentError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 429) { 0164 // rate limiting error 0165 executor->cooldownTime *= 2; 0166 qDebug() << "rate limit error, increasing cooldown time to" << executor->cooldownTime.count() << "seconds"; 0167 m_tasks.push_back(std::move(executor->task)); 0168 } else if (reply->error() == QNetworkReply::ContentNotFoundError && !executor->task->forceReload) { 0169 // cache miss, retry from network 0170 executor->task->forceReload = true; 0171 m_tasks.push_back(std::move(executor->task)); 0172 } else if (reply->error() != QNetworkReply::NoError) { 0173 // TODO disable affected executors here and reschedule the failed task, rather than cancelling entirely 0174 qDebug() << reply->error() << reply->errorString() << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) << reply->readAll(); 0175 query->m_error = OverpassQuery::NetworkError; 0176 cancelQuery(query); 0177 } else { 0178 const auto queryError = query->processReply(reply); 0179 // on query timeout, break up the task in 4 sub-tasks, if we are allowed to 0180 if (queryError == OverpassQuery::QueryTimeout 0181 && executor->task->bbox.width() > query->minimumTileSize().width() 0182 && executor->task->bbox.height() > query->minimumTileSize().height()) 0183 { 0184 qDebug() << "Splitting task due to query timeout:" << executor->task->bbox; 0185 const auto xTileSize = executor->task->bbox.width() / 2.0; 0186 const auto yTileSize = executor->task->bbox.height() / 2.0; 0187 for (auto x = 0; x < 2; ++x) { 0188 for (auto y = 0; y < 2; ++y) { 0189 auto task = std::make_unique<OverpassQueryTask>(); 0190 task->query = query; 0191 task->bbox = { executor->task->bbox.x() + x * xTileSize, executor->task->bbox.y() + y * yTileSize, xTileSize, yTileSize }; 0192 m_tasks.push_back(std::move(task)); 0193 } 0194 } 0195 } 0196 else if (queryError != OverpassQuery::NoError) { 0197 if (executor->task->forceReload) { 0198 query->m_error = queryError; 0199 cancelQuery(query); 0200 } else { 0201 // query error in cached result, retry 0202 executor->task->forceReload = true; 0203 m_tasks.push_back(std::move(executor->task)); 0204 } 0205 } 0206 } 0207 0208 // free the executor for the next query 0209 executor->task.reset(); 0210 executor->nextSlot = QDateTime::currentDateTimeUtc().addSecs(executor->cooldownTime.count()); 0211 0212 checkQueryFinished(query); 0213 executeTasks(); 0214 } 0215 0216 void OverpassQueryManagerPrivate::checkQueryFinished(OverpassQuery *query) const 0217 { 0218 if (std::any_of(m_executors.begin(), m_executors.end(), [query](const auto &executor) { return executor.task && executor.task->query == query; }) 0219 || std::any_of(m_tasks.begin(), m_tasks.end(), [query](const auto &task) { return task->query == query; })) 0220 return; 0221 Q_EMIT query->finished(); 0222 } 0223 0224 void OverpassQueryManagerPrivate::cancelQuery(OverpassQuery *query) 0225 { 0226 qDebug() << "cancelling query..."; 0227 m_tasks.erase(std::remove_if(m_tasks.begin(), m_tasks.end(), [query](const auto &task) { return task->query == query; }), m_tasks.end()); 0228 checkQueryFinished(query); 0229 } 0230 0231 #include "moc_overpassquerymanager.cpp"