map/loader/tilecache.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
0003
0004     SPDX-License-Identifier: LGPL-2.0-or-later
0005 */
0006
0007 #include "tilecache_p.h"
0008 #include "logging.h"
0009
0010 #include <osm/datatypes.h>
0011 #include <osm/geomath.h>
0012
0013 #include <QCoreApplication>
0014 #include <QDir>
0015 #include <QDirIterator>
0016 #include <QFile>
0017 #include <QFileInfo>
0018 #include <QNetworkAccessManager>
0019 #include <QNetworkReply>
0020 #include <QStandardPaths>
0021 #include <QUrl>
0022
0023 #include <cmath>
0024
0025 using namespace KOSMIndoorMap;
0026
0027 enum {
0028     DefaultCacheDays = 14,
0029 };
0030
0031 Tile Tile::fromCoordinate(double lat, double lon, uint8_t z)
0032 {
0033     Tile t;
0034     t.x = std::floor((lon + 180.0) / 360.0 * (1 << z));
0035     const auto latrad = OSM::degToRad(lat);
0036     t.y = std::floor((1.0 - std::asinh(std::tan(latrad)) / M_PI) / 2.0 * (1 << z));
0037     t.z = z;
0038     return t;
0039 }
0040
0041 OSM::Coordinate Tile::topLeft() const
0042 {
0043     const auto lon = x / (double)(1 << z) * 360.0 - 180.0;
0044
0045     const auto n = M_PI - 2.0 * M_PI * y / (double)(1 << z);
0046     const auto lat = OSM::radToDeg(std::atan(0.5 * (std::exp(n) - std::exp(-n))));
0047
0048     return OSM::Coordinate(lat, lon);
0049 }
0050
0051 OSM::BoundingBox Tile::boundingBox() const
0052 {
0053     Tile bottomRight = *this;
0054     ++bottomRight.x;
0055     ++bottomRight.y;
0056
0057     const auto tl = topLeft();
0058     const auto br = bottomRight.topLeft();
0059
0060     return OSM::BoundingBox(OSM::Coordinate(br.latitude, tl.longitude), OSM::Coordinate(tl.latitude, br.longitude));
0061 }
0062
0063 Tile Tile::topLeftAtZ(uint8_t z) const
0064 {
0065     if (z == this->z) {
0066         return *this;
0067     }
0068     if (z < this->z) {
0069         return Tile{ x / (1 << (this->z - z)), y / (1 << (this->z - z)), z};
0070     }
0071     return Tile{ x * (1 << (z - this->z )), y * (1 << (z - this->z)), z};
0072 }
0073
0074 Tile Tile::bottomRightAtZ(uint8_t z) const
0075 {
0076     if (z <= this->z) {
0077         return topLeftAtZ(z);
0078     }
0079     const auto deltaZ = z - this->z;
0080     const auto deltaWidth = 1 << deltaZ;
0081     return Tile{ x * deltaWidth + deltaWidth - 1, y * deltaWidth + deltaWidth - 1, z};
0082 }
0083
0084 TileCache::TileCache(QObject *parent)
0085     : QObject(parent)
0086     , m_nam(new QNetworkAccessManager(this))
0087 {
0088     m_nam->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
0089     m_nam->enableStrictTransportSecurityStore(true, QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.osm/hsts/"));
0090     m_nam->setStrictTransportSecurityEnabled(true);
0091 }
0092
0093 TileCache::~TileCache() = default;
0094
0095 QString TileCache::cachedTile(Tile tile) const
0096 {
0097     const auto p = cachePath(tile);
0098     if (QFile::exists(p)) {
0099         return p;
0100     }
0101     return {};
0102 }
0103
0104 void TileCache::ensureCached(Tile tile)
0105 {
0106     const auto t = cachedTile(tile);
0107     if (t.isEmpty()) {
0108         downloadTile(tile);
0109         return;
0110     }
0111
0112     if (tile.ttl.isValid()) {
0113         updateTtl(t, tile.ttl);
0114     }
0115 }
0116
0117 void TileCache::downloadTile(Tile tile)
0118 {
0119     m_pendingDownloads.push_back(tile);
0120     downloadNext();
0121 }
0122
0123 QString TileCache::cachePath(Tile tile) const
0124 {
0125     QString base;
0126     if (!qEnvironmentVariableIsSet("KOSMINDOORMAP_CACHE_PATH")) {
0127         base = QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation)
0128             + QLatin1String("/org.kde.osm/vectorosm/");
0129     } else {
0130         base = qEnvironmentVariable("KOSMINDOORMAP_CACHE_PATH");
0131     }
0132
0133     return base
0134         + QString::number(tile.z) + QLatin1Char('/')
0135         + QString::number(tile.x) + QLatin1Char('/')
0136         + QString::number(tile.y) + QLatin1String(".o5m");
0137 }
0138
0139 void TileCache::downloadNext()
0140 {
0141     if (m_output.isOpen() || m_pendingDownloads.empty()) {
0142         return;
0143     }
0144
0145     const auto tile = m_pendingDownloads.front();
0146     m_pendingDownloads.pop_front();
0147
0148     QFileInfo fi(cachePath(tile));
0149     QDir().mkpath(fi.absolutePath());
0150     m_output.setFileName(fi.absoluteFilePath() + QLatin1String(".part"));
0151     if (!m_output.open(QFile::WriteOnly)) {
0152         qCWarning(Log) << m_output.fileName() << m_output.errorString();
0153         return;
0154     }
0155
0156     QUrl url;
0157     if (qEnvironmentVariableIsSet("KOSMINDOORMAP_TILESERVER")) {
0158         url = QUrl(qEnvironmentVariable("KOSMINDOORMAP_TILESERVER"));
0159     } else {
0160         url.setScheme(QStringLiteral("https"));
0161         url.setHost(QStringLiteral("maps.kde.org"));
0162         url.setPath(QStringLiteral("/earth/vectorosm/v1/"));
0163     }
0164
0165     url.setPath(url.path() + QString::number(tile.z) + QLatin1Char('/')
0166         + QString::number(tile.x) + QLatin1Char('/')
0167         + QString::number(tile.y) + QLatin1String(".o5m"));
0168
0169     QNetworkRequest req(url);
0170     req.setAttribute(QNetworkRequest::Http2AllowedAttribute, true);
0171     req.setHeader(QNetworkRequest::UserAgentHeader, (QCoreApplication::applicationName() + QLatin1Char('/') + QCoreApplication::applicationVersion()).toUtf8());
0172     auto reply = m_nam->get(req);
0173     connect(reply, &QNetworkReply::readyRead, this, [this, reply]() { dataReceived(reply); });
0174     connect(reply, &QNetworkReply::finished, this, [this, reply, tile]() { downloadFinished(reply, tile); });
0175     connect(reply, &QNetworkReply::sslErrors, this, [reply](const auto &sslErrors) { reply->setProperty("_ssl_errors", QVariant::fromValue(sslErrors)); });
0176 }
0177
0178 void TileCache::dataReceived(QNetworkReply *reply)
0179 {
0180     m_output.write(reply->read(reply->bytesAvailable()));
0181 }
0182
0183 void TileCache::downloadFinished(QNetworkReply* reply, Tile tile)
0184 {
0185     reply->deleteLater();
0186     m_output.close();
0187
0188     if (reply->error() != QNetworkReply::NoError) {
0189         qCWarning(Log) << reply->errorString() << reply->url();
0190         m_output.remove();
0191         if (reply->error() == QNetworkReply::SslHandshakeFailedError) {
0192             const auto sslErrors = reply->property("_ssl_errors").value<QList<QSslError>>();
0193             QStringList errorStrings;
0194             errorStrings.reserve(sslErrors.size());
0195             std::transform(sslErrors.begin(), sslErrors.end(), std::back_inserter(errorStrings), [](const auto &e) { return e.errorString(); });
0196             qCWarning(Log) << errorStrings;
0197             Q_EMIT tileError(tile, reply->errorString() + QLatin1String(" (") + errorStrings.join(QLatin1String(", ")) + QLatin1Char(')'));
0198         } else {
0199             Q_EMIT tileError(tile, reply->errorString());
0200         }
0201         downloadNext();
0202         return;
0203     }
0204
0205     const auto t = cachePath(tile);
0206     m_output.rename(t);
0207     if (tile.ttl.isValid()) {
0208         updateTtl(t, std::max(QDateTime::currentDateTimeUtc().addDays(1), tile.ttl));
0209     } else {
0210         updateTtl(t, QDateTime::currentDateTimeUtc().addDays(DefaultCacheDays));
0211     }
0212
0213     Q_EMIT tileLoaded(tile);
0214     downloadNext();
0215 }
0216
0217 int TileCache::pendingDownloads() const
0218 {
0219     return m_pendingDownloads.size() + (m_output.isOpen() ? 1 : 0);
0220 }
0221
0222 void TileCache::cancelPending()
0223 {
0224     m_pendingDownloads.clear();
0225 }
0226
0227 static void expireRecursive(const QString &path)
0228 {
0229     QDirIterator it(path, QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
0230     while (it.hasNext()) {
0231         it.next();
0232
0233         if (it.fileInfo().isDir()) {
0234             expireRecursive(it.filePath());
0235             if (QDir(it.filePath()).isEmpty()) {
0236                 qCDebug(Log) << "removing empty tile directory" << it.fileName();
0237                 QDir(path).rmdir(it.filePath());
0238             }
0239         } else if (it.fileInfo().lastModified() < QDateTime::currentDateTimeUtc()) {
0240             qCDebug(Log) << "removing expired tile" << it.filePath();
0241             QDir(path).remove(it.filePath());
0242         }
0243     }
0244 }
0245 void TileCache::expire()
0246 {
0247     const QString base = QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.osm/vectorosm/");
0248     expireRecursive(base);
0249 }
0250
0251 void TileCache::updateTtl(const QString &filePath, const QDateTime &ttl)
0252 {
0253     QFile f(filePath);
0254     f.open(QFile::WriteOnly | QFile::Append);
0255     f.setFileTime(std::max(f.fileTime(QFileDevice::FileModificationTime), ttl), QFile::FileModificationTime);
0256 }
0257
0258 #include "moc_tilecache_p.cpp"