File indexing completed on 2023-10-03 07:01:11
0001 // SPDX-License-Identifier: LGPL-2.1-or-later 0002 // 0003 // SPDX-FileCopyrightText: 2016 Dennis Nienhüser <nienhueser@kde.org> 0004 // 0005 0006 #include "TileDirectory.h" 0007 #include "TileIterator.h" 0008 #include <GeoDataDocumentWriter.h> 0009 #include "MarbleZipReader.h" 0010 #include <GeoDataLatLonAltBox.h> 0011 #include "PeakAnalyzer.h" 0012 #include "TileCoordsPyramid.h" 0013 #include "StyleBuilder.h" 0014 0015 #include <QFileInfo> 0016 #include <QDebug> 0017 #include <QProcess> 0018 #include <QDir> 0019 #include <QUrl> 0020 #include <QNetworkRequest> 0021 #include <QNetworkReply> 0022 #include <QTemporaryFile> 0023 #include <QThread> 0024 0025 #include <iostream> 0026 #include <iomanip> 0027 0028 using namespace std; 0029 0030 namespace Marble { 0031 0032 QMap<int, TagsFilter::Tags> TileDirectory::m_tags; 0033 0034 TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, int maxZoomLevel) : 0035 m_cacheDir(cacheDir), 0036 m_manager(manager), 0037 m_tileType(tileType), 0038 m_landmassFile("land-polygons-split-4326.zip"), 0039 m_maxZoomLevel(maxZoomLevel) 0040 { 0041 if (m_tileType == Landmass) { 0042 m_zoomLevel = 7; 0043 m_baseDir = QString("%1/landmass/%2").arg(cacheDir).arg(m_zoomLevel); 0044 QString const landmassDir = QString("%1/land-polygons-split-4326").arg(cacheDir); 0045 m_inputFile = QString("%1/land_polygons.shp").arg(landmassDir); 0046 auto const landmassZip = QString("%1/%2").arg(m_cacheDir).arg(m_landmassFile); 0047 if (!QFileInfo(landmassZip).exists()) { 0048 QString const url = QString("https://osmdata.openstreetmap.de/download/%1").arg(m_landmassFile); 0049 download(url, landmassZip); 0050 } 0051 0052 if (!QFileInfo(landmassDir).exists()) { 0053 MarbleZipReader unzip(landmassZip); 0054 if (!unzip.extractAll(m_cacheDir)) { 0055 qWarning() << "Failed to extract" << landmassZip << "to" << m_cacheDir; 0056 } 0057 } 0058 } else { 0059 m_zoomLevel = 10; 0060 m_baseDir = QString("%1/osm/%2").arg(cacheDir).arg(m_zoomLevel); 0061 } 0062 QDir().mkpath(m_baseDir); 0063 } 0064 0065 TileDirectory::TileDirectory(const QString &cacheDir, const QString &osmxFile, ParsingRunnerManager &manager, int maxZoomLevel, int loadZoomLevel) : 0066 m_cacheDir(cacheDir), 0067 m_osmxFile(osmxFile), 0068 m_manager(manager), 0069 m_zoomLevel(loadZoomLevel), 0070 m_tileType(OpenStreetMap), 0071 m_maxZoomLevel(maxZoomLevel) 0072 { 0073 } 0074 0075 TileId TileDirectory::tileFor(int zoomLevel, int tileX, int tileY) const 0076 { 0077 int const zoomDiff = zoomLevel - m_zoomLevel; 0078 int const x = tileX >> zoomDiff; 0079 int const y = tileY >> zoomDiff; 0080 return TileId(QString(), m_zoomLevel, x, y); 0081 } 0082 0083 QSharedPointer<GeoDataDocument> TileDirectory::load(int zoomLevel, int tileX, int tileY) 0084 { 0085 auto const tile = tileFor(zoomLevel, tileX, tileY); 0086 if (tile.x() == m_tileX && tile.y() == m_tileY) { 0087 return m_landmass; 0088 } 0089 m_tileX = tile.x(); 0090 m_tileY = tile.y(); 0091 0092 if (!m_osmxFile.isEmpty()) { 0093 const auto tileBox = m_tileProjection.geoCoordinates(tile); 0094 const QString bbox = QString::number(tileBox.south(GeoDataCoordinates::Degree)) 0095 + QLatin1Char(',') + QString::number(tileBox.west(GeoDataCoordinates::Degree)) 0096 + QLatin1Char(',') + QString::number(tileBox.north(GeoDataCoordinates::Degree)) 0097 + QLatin1Char(',') + QString::number(tileBox.east(GeoDataCoordinates::Degree)); 0098 0099 // TODO the following could be optimized by directly reading via OSMX API 0100 QTemporaryFile tempPbfFile(m_cacheDir + "/tmp/XXXXXX.osm.pbf"); 0101 if (!tempPbfFile.open()) { 0102 qCritical() << "Failed to open temporary file!" << tempPbfFile.errorString() << m_cacheDir; 0103 return {}; 0104 } 0105 0106 QProcess osmx; 0107 osmx.start("osmx", QStringList({ "extract" , (m_cacheDir + QLatin1Char('/') + m_osmxFile), tempPbfFile.fileName(), "--noUserData", "--bbox", bbox })); 0108 osmx.waitForFinished(5*60*1000); 0109 if (osmx.exitCode() != 0) { 0110 qWarning() << osmx.readAllStandardError(); 0111 qWarning() << "osmx failed: " << osmx.errorString() << osmx.exitStatus() << osmx.exitCode(); 0112 return {}; 0113 } 0114 0115 m_landmass = open(tempPbfFile.fileName(), m_manager); 0116 } else { 0117 QString const filename = QString("%1/%2/%3.%4").arg(m_baseDir).arg(tile.x()).arg(tile.y()).arg("o5m"); 0118 m_landmass = open(filename, m_manager); 0119 } 0120 0121 if (m_landmass) { 0122 PeakAnalyzer::determineZoomLevel(m_landmass->placemarkList()); 0123 } 0124 return m_landmass; 0125 } 0126 0127 void TileDirectory::setInputFile(const QString &filename) 0128 { 0129 m_inputFile = filename; 0130 0131 if (m_tileType == OpenStreetMap) { 0132 QUrl url = QUrl(filename); 0133 if (url.scheme().isEmpty()) { 0134 // local file 0135 m_boundingBox = boundingBox(m_inputFile); 0136 } else { 0137 // remote file: check if already downloaded 0138 QFileInfo cacheFile = QString("%1/%2").arg(m_cacheDir).arg(url.fileName()); 0139 if (!cacheFile.exists()) { 0140 download(filename, cacheFile.absoluteFilePath()); 0141 } 0142 m_inputFile = cacheFile.absoluteFilePath(); 0143 0144 QString polyFile = QUrl(filename).fileName(); 0145 polyFile.replace("-latest.osm.pbf", ".poly"); 0146 polyFile.replace(".osm.pbf", ".poly"); 0147 polyFile.replace(".pbf", ".poly"); 0148 QString poly = QString("%1/%2").arg(url.adjusted(QUrl::RemoveFilename).toString()).arg(polyFile); 0149 QString const polyTarget = QString("%1/%2").arg(m_cacheDir).arg(polyFile); 0150 if (!QFileInfo(polyTarget).exists()) { 0151 download(poly, polyTarget); 0152 } 0153 setBoundingPolygon(polyTarget); 0154 } 0155 } 0156 } 0157 0158 GeoDataDocument* TileDirectory::clip(int zoomLevel, int tileX, int tileY) 0159 { 0160 QSharedPointer<GeoDataDocument> oldMap = m_landmass; 0161 load(zoomLevel, tileX, tileY); 0162 if (!m_clipper || oldMap != m_landmass || m_tagZoomLevel != zoomLevel) { 0163 setTagZoomLevel(zoomLevel); 0164 GeoDataDocument* input = m_tagsFilter ? m_tagsFilter->accepted() : m_landmass.data(); 0165 if (input) { 0166 m_clipper = QSharedPointer<VectorClipper>(new VectorClipper(input, m_maxZoomLevel)); 0167 } 0168 } 0169 return m_clipper ? m_clipper->clipTo(zoomLevel, tileX, tileY) : nullptr; 0170 } 0171 0172 QString TileDirectory::name() const 0173 { 0174 return QString("%1/%2/%3").arg(m_zoomLevel).arg(m_tileX).arg(m_tileY); 0175 } 0176 0177 QSharedPointer<GeoDataDocument> TileDirectory::open(const QString &filename, ParsingRunnerManager &manager) 0178 { 0179 // Timeout is set to 10 min. If the file is reaaally huge, set it to something bigger. 0180 GeoDataDocument* map = manager.openFile(filename, DocumentRole::MapDocument, 600000); 0181 if(map == nullptr) { 0182 qWarning() << "File" << filename << "couldn't be loaded."; 0183 } 0184 QSharedPointer<GeoDataDocument> result = QSharedPointer<GeoDataDocument>(map); 0185 return result; 0186 } 0187 0188 TagsFilter::Tags TileDirectory::tagsFilteredIn(int zoomLevel) const 0189 { 0190 if (m_tags.isEmpty()) { 0191 QSet<GeoDataPlacemark::GeoDataVisualCategory> categories; 0192 for (int i=GeoDataPlacemark::PlaceCity; i<GeoDataPlacemark::LastIndex; ++i) { 0193 categories << GeoDataPlacemark::GeoDataVisualCategory(i); 0194 } 0195 0196 auto const tagMap = StyleBuilder::osmTagMapping(); 0197 for (auto category: categories) { 0198 for (auto iter=tagMap.begin(), end=tagMap.end(); iter != end; ++iter) { 0199 if (iter.value() == category) { 0200 int zoomLevel = StyleBuilder::minimumZoomLevel(category); 0201 if (zoomLevel < 17) { 0202 m_tags[zoomLevel] << iter.key(); 0203 } 0204 } 0205 } 0206 } 0207 } 0208 0209 TagsFilter::Tags result; 0210 for (auto iter = m_tags.begin(), end = m_tags.end(); iter != end && iter.key() <= zoomLevel+1; ++iter) { 0211 result << iter.value(); 0212 } 0213 return result; 0214 } 0215 0216 void TileDirectory::setTagZoomLevel(int zoomLevel) 0217 { 0218 m_tagZoomLevel = zoomLevel; 0219 if (m_tileType == OpenStreetMap) { 0220 if (m_tagZoomLevel < 17) { 0221 auto const tags = tagsFilteredIn(m_tagZoomLevel); 0222 m_tagsFilter = QSharedPointer<TagsFilter>(new TagsFilter(m_landmass.data(), tags, TagsFilter::FilterRailwayService)); 0223 } else { 0224 m_tagsFilter.clear(); 0225 } 0226 } 0227 } 0228 0229 void TileDirectory::download(const QString &url, const QString &target) 0230 { 0231 m_download = QSharedPointer<Download>(new Download); 0232 m_download->target = target; 0233 m_download->reply = m_downloadManager.get(QNetworkRequest(QUrl(url))); 0234 connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), m_download.data(), SLOT(updateProgress(qint64,qint64))); 0235 connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(updateProgress())); 0236 QEventLoop loop; 0237 connect(m_download->reply, SIGNAL(finished()), &loop, SLOT(quit())); 0238 loop.exec(); 0239 cout << endl; 0240 } 0241 0242 QString TileDirectory::osmFileFor(const TileId &tileId) const 0243 { 0244 QString const outputDir = QString("%1/osm/%2/%3").arg(m_cacheDir).arg(tileId.zoomLevel()).arg(tileId.x()); 0245 return QString("%1/%2.o5m").arg(outputDir).arg(tileId.y()); 0246 } 0247 0248 void TileDirectory::printProgress(double progress, int barWidth) 0249 { 0250 int const position = barWidth * progress; 0251 cout << " [" << string(position, '=') << ">"; 0252 cout << string(barWidth-position, ' ') << "] " << std::right << setw(3) << int(progress * 100.0) << "%"; 0253 } 0254 0255 GeoDataLatLonBox TileDirectory::boundingBox() const 0256 { 0257 return m_boundingBox; 0258 } 0259 0260 void TileDirectory::setBoundingBox(const GeoDataLatLonBox &boundingBox) 0261 { 0262 m_boundingBox = boundingBox; 0263 } 0264 0265 void TileDirectory::setBoundingPolygon(const QString &file) 0266 { 0267 m_boundingPolygon.clear(); 0268 QFile input(file); 0269 QString country = "Unknown"; 0270 if ( input.open( QFile::ReadOnly ) ) { 0271 QTextStream stream( &input ); 0272 country = stream.readLine(); 0273 double lat( 0.0 ), lon( 0.0 ); 0274 GeoDataLinearRing box; 0275 while ( !stream.atEnd() ) { 0276 bool inside = true; 0277 QString line = stream.readLine().trimmed(); 0278 QStringList entries = line.split( QLatin1Char( ' ' ), QString::SkipEmptyParts ); 0279 if ( entries.size() == 1 ) { 0280 if (entries.first() == QLatin1String("END") && inside) { 0281 inside = false; 0282 if (!box.isEmpty()) { 0283 m_boundingPolygon << box; 0284 box = GeoDataLinearRing(); 0285 } 0286 } else if (entries.first() == QLatin1String("END") && !inside) { 0287 qDebug() << "END not expected here"; 0288 } else if ( entries.first().startsWith( QLatin1String( "!" ) ) ) { 0289 qDebug() << "Warning: Negative polygons not supported, skipping"; 0290 } else { 0291 //int number = entries.first().toInt(); 0292 inside = true; 0293 } 0294 } else if ( entries.size() == 2 ) { 0295 lon = entries.first().toDouble(); 0296 lat = entries.last().toDouble(); 0297 GeoDataCoordinates point( lon, lat, 0.0, GeoDataCoordinates::Degree ); 0298 box << point; 0299 } else { 0300 qDebug() << "Warning: Ignoring line in" << file 0301 << "with" << entries.size() << "fields:" << line; 0302 } 0303 } 0304 } 0305 0306 if (!m_boundingPolygon.isEmpty()) { 0307 m_boundingBox = GeoDataLatLonBox::fromLineString(m_boundingPolygon.first()); 0308 for (int i=1, n=m_boundingPolygon.size(); i<n; ++i) { 0309 m_boundingBox |= GeoDataLatLonBox::fromLineString(m_boundingPolygon[i]); 0310 } 0311 } else { 0312 m_boundingBox = boundingBox(m_inputFile); 0313 } 0314 } 0315 0316 void TileDirectory::createTiles() const 0317 { 0318 if (m_tileType == OpenStreetMap) { 0319 createOsmTiles(); 0320 return; 0321 } 0322 0323 QSharedPointer<GeoDataDocument> map; 0324 QSharedPointer<VectorClipper> clipper; 0325 TileIterator iter(m_boundingBox, m_zoomLevel); 0326 qint64 count = 0; 0327 for(auto const &tileId: iter) { 0328 ++count; 0329 QString const outputDir = QString("%1/%2").arg(m_baseDir).arg(tileId.x()); 0330 QString const outputFile = QString("%1/%2.o5m").arg(outputDir).arg(tileId.y()); 0331 if (QFileInfo(outputFile).exists()) { 0332 continue; 0333 } 0334 0335 printProgress(count / double(iter.total())); 0336 cout << " Creating landmass cache tile " << count << "/" << iter.total() << " ("; 0337 cout << m_zoomLevel << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r'; 0338 cout.flush(); 0339 0340 QDir().mkpath(outputDir); 0341 if (!clipper) { 0342 map = open(m_inputFile, m_manager); 0343 if (!map) { 0344 qCritical() << "Failed to open " << m_inputFile << ". This can happen when Marble was compiled without shapelib (libshp), when the system has too little memory (RAM + swap need to be at least 8G), or when the download of the landmass data file failed."; 0345 } 0346 clipper = QSharedPointer<VectorClipper>(new VectorClipper(map.data(), m_zoomLevel)); 0347 } 0348 std::unique_ptr<GeoDataDocument> tile(clipper->clipTo(m_zoomLevel, tileId.x(), tileId.y())); 0349 if (!GeoDataDocumentWriter::write(outputFile, *tile)) { 0350 qWarning() << "Failed to write tile" << outputFile; 0351 } 0352 } 0353 printProgress(1.0); 0354 cout << " landmass cache tiles complete." << string(20, ' ') << endl; 0355 } 0356 0357 void TileDirectory::createOsmTiles() const 0358 { 0359 const GeoSceneMercatorTileProjection tileProjection; 0360 const QRect rect = tileProjection.tileIndexes(m_boundingBox, m_zoomLevel); 0361 TileCoordsPyramid pyramid(0, m_zoomLevel); 0362 pyramid.setBottomLevelCoords(rect); 0363 0364 qint64 const maxCount = pyramid.tilesCount(); 0365 QMap<int,QVector<TileId> > tileLevels; 0366 for (int zoomLevel=0; zoomLevel <= m_zoomLevel; ++zoomLevel) { 0367 QRect const rect = pyramid.coords(zoomLevel); 0368 if (zoomLevel < m_zoomLevel && rect.width()*rect.height() < 2) { 0369 continue; 0370 } 0371 0372 TileIterator iter(m_boundingBox, zoomLevel); 0373 for(auto const &tileId: iter) { 0374 tileLevels[zoomLevel] << TileId(0, zoomLevel, tileId.x(), tileId.y()); 0375 } 0376 } 0377 0378 bool hasAllTiles = true; 0379 for (auto const &tileId: tileLevels[m_zoomLevel]) { 0380 auto const outputFile = osmFileFor(tileId); 0381 if (!QFileInfo(outputFile).exists()) { 0382 hasAllTiles = false; 0383 break; 0384 } 0385 } 0386 0387 bool first = true; 0388 if (!hasAllTiles) { 0389 qint64 count = 0; 0390 for (auto const &tiles: tileLevels) { 0391 for (auto const &tileId: tiles) { 0392 ++count; 0393 QString const inputFile = first ? m_inputFile : QString("%1/osm/%2/%3/%4.o5m"). 0394 arg(m_cacheDir).arg(tileId.zoomLevel()-1).arg(tileId.x()>>1).arg(tileId.y()>>1); 0395 QString const outputFile = osmFileFor(tileId); 0396 if (QFileInfo(outputFile).exists()) { 0397 continue; 0398 } 0399 0400 printProgress(count / double(maxCount)); 0401 cout << " Creating osm cache tile " << count << "/" << maxCount << " ("; 0402 cout << tileId.zoomLevel() << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r'; 0403 cout.flush(); 0404 0405 QDir().mkpath(QFileInfo(outputFile).absolutePath()); 0406 QString const output = QString("-o=%1").arg(outputFile); 0407 0408 const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tileId.zoomLevel(), tileId.x(), tileId.y()); 0409 0410 double const minLon = tileBoundary.west(GeoDataCoordinates::Degree); 0411 double const maxLon = tileBoundary.east(GeoDataCoordinates::Degree); 0412 double const maxLat = tileBoundary.north(GeoDataCoordinates::Degree); 0413 double const minLat = tileBoundary.south(GeoDataCoordinates::Degree); 0414 QString const bbox = QString("-b=%1,%2,%3,%4").arg(minLon).arg(minLat).arg(maxLon).arg(maxLat); 0415 QProcess osmconvert; 0416 osmconvert.start("osmconvert", QStringList() << "--drop-author" << "--drop-version" 0417 << "--complete-ways" << "--complex-ways" << bbox << output << inputFile); 0418 osmconvert.waitForFinished(10*60*1000); 0419 if (osmconvert.exitCode() != 0) { 0420 qWarning() << osmconvert.readAllStandardError(); 0421 qWarning() << "osmconvert failed: " << osmconvert.errorString(); 0422 } 0423 } 0424 first = false; 0425 } 0426 } 0427 0428 tileLevels.remove(m_zoomLevel); 0429 for (auto const &tiles: tileLevels) { 0430 for (auto const &tileId: tiles) { 0431 QFile::remove(osmFileFor(tileId)); 0432 } 0433 } 0434 0435 printProgress(1.0); 0436 cout << " osm cache tiles complete." << string(20, ' ') << endl; 0437 0438 } 0439 0440 int TileDirectory::innerNodes(const TileId &tile) const 0441 { 0442 const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tile.zoomLevel(), tile.x(), tile.y()); 0443 0444 double const west = tileBoundary.west(); 0445 double const east = tileBoundary.east(); 0446 double const north = tileBoundary.north(); 0447 double const south = tileBoundary.south(); 0448 QVector<GeoDataCoordinates> bounds; 0449 bounds << GeoDataCoordinates(west, north); 0450 bounds << GeoDataCoordinates(east, north); 0451 bounds << GeoDataCoordinates(east, south); 0452 bounds << GeoDataCoordinates(west, south); 0453 0454 int innerNodes = 0; 0455 if (m_boundingPolygon.isEmpty()) { 0456 for(auto const &coordinate: bounds) { 0457 if (m_boundingBox.contains(coordinate)) { 0458 ++innerNodes; 0459 } 0460 } 0461 return innerNodes; 0462 } 0463 0464 for(auto const &coordinate: bounds) { 0465 for(auto const &ring: m_boundingPolygon) { 0466 if (ring.contains(coordinate)) { 0467 ++innerNodes; 0468 } 0469 } 0470 } 0471 return innerNodes; 0472 } 0473 0474 void TileDirectory::updateProgress() 0475 { 0476 double const progress = m_download->total > 0 ? m_download->received / double(m_download->total) : 0.0; 0477 printProgress(progress); 0478 0479 cout << " "; 0480 cout << std::fixed << std::setprecision(1) << m_download->received / 1000000.0 << '/'; 0481 cout << std::fixed << std::setprecision(1) << m_download->total / 1000000.0 << " MB"; 0482 0483 cout << " Downloading " << m_download->reply->url().fileName().toStdString(); 0484 0485 cout << string(20, ' ') << '\r'; 0486 cout.flush(); 0487 } 0488 0489 void TileDirectory::handleFinishedDownload(const QString &filename, const QString &id) 0490 { 0491 qDebug() << "File " << filename << "(" << id << ") has been downloaded."; 0492 } 0493 0494 GeoDataLatLonBox TileDirectory::boundingBox(const QString &filename) const 0495 { 0496 QProcess osmconvert; 0497 osmconvert.start("osmconvert", QStringList() << "--out-statistics" << filename); 0498 osmconvert.waitForFinished(10*60*1000); 0499 QStringList const output = QString(osmconvert.readAllStandardOutput()).split('\n'); 0500 GeoDataLatLonBox boundingBox; 0501 for(QString const &line: output) { 0502 if (line.startsWith("lon min:")) { 0503 boundingBox.setWest(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0504 } else if (line.startsWith("lon max")) { 0505 boundingBox.setEast(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0506 } else if (line.startsWith("lat min:")) { 0507 boundingBox.setSouth(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0508 } else if (line.startsWith("lat max:")) { 0509 boundingBox.setNorth(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0510 } 0511 } 0512 return boundingBox; 0513 } 0514 0515 void Download::updateProgress(qint64 received_, qint64 total_) 0516 { 0517 received = received_; 0518 total = total_; 0519 0520 QString const tempFile = QString("%1.download").arg(target); 0521 if (!m_file.isOpen()) { 0522 m_file.setFileName(tempFile); 0523 m_file.open(QFile::WriteOnly); 0524 } 0525 m_file.write(reply->readAll()); 0526 0527 if (reply->isFinished()) { 0528 m_file.flush(); 0529 m_file.close(); 0530 QFile::rename(tempFile, target); 0531 } 0532 } 0533 0534 } 0535 0536 #include "moc_TileDirectory.cpp"