File indexing completed on 2024-04-21 03:50:54
0001 // SPDX-License-Identifier: LGPL-2.1-or-later 0002 // 0003 // SPDX-FileCopyrightText: 2016 Dennis Nienhüser <nienhueser@kde.org> 0004 // 0005 0006 #include "TileDirectory.h" 0007 #include "TileIterator.h" 0008 #include <GeoDataDocumentWriter.h> 0009 #include "MarbleZipReader.h" 0010 #include <GeoDataLatLonAltBox.h> 0011 #include "PeakAnalyzer.h" 0012 #include "TileCoordsPyramid.h" 0013 #include "StyleBuilder.h" 0014 0015 #include <QFileInfo> 0016 #include <QDebug> 0017 #include <QProcess> 0018 #include <QDir> 0019 #include <QUrl> 0020 #include <QNetworkRequest> 0021 #include <QNetworkReply> 0022 #include <QTemporaryFile> 0023 #include <QThread> 0024 0025 #include <iostream> 0026 #include <iomanip> 0027 0028 using namespace std; 0029 0030 namespace Marble { 0031 0032 QMap<int, TagsFilter::Tags> TileDirectory::m_tags; 0033 0034 TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, int maxZoomLevel) : 0035 m_cacheDir(cacheDir), 0036 m_manager(manager), 0037 m_tileType(tileType), 0038 m_landmassFile("land-polygons-split-4326.zip"), 0039 m_maxZoomLevel(maxZoomLevel) 0040 { 0041 if (m_tileType == Landmass) { 0042 m_zoomLevel = 7; 0043 m_baseDir = QString("%1/landmass/%2").arg(cacheDir).arg(m_zoomLevel); 0044 } else { 0045 m_zoomLevel = 10; 0046 m_baseDir = QString("%1/osm/%2").arg(cacheDir).arg(m_zoomLevel); 0047 } 0048 QDir().mkpath(m_baseDir); 0049 } 0050 0051 TileDirectory::TileDirectory(const QString &cacheDir, const QString &osmxFile, ParsingRunnerManager &manager, int maxZoomLevel, int loadZoomLevel, InputType inputType) : 0052 m_cacheDir(cacheDir), 0053 m_osmxFile(osmxFile), 0054 m_manager(manager), 0055 m_zoomLevel(loadZoomLevel), 0056 m_tileType(OpenStreetMap), 0057 m_inputType(inputType), 0058 m_maxZoomLevel(maxZoomLevel) 0059 { 0060 } 0061 0062 TileId TileDirectory::tileFor(int zoomLevel, int tileX, int tileY) const 0063 { 0064 int const zoomDiff = zoomLevel - m_zoomLevel; 0065 int const x = tileX >> zoomDiff; 0066 int const y = tileY >> zoomDiff; 0067 return TileId(QString(), m_zoomLevel, x, y); 0068 } 0069 0070 QSharedPointer<GeoDataDocument> TileDirectory::load(int zoomLevel, int tileX, int tileY) 0071 { 0072 auto const tile = tileFor(zoomLevel, tileX, tileY); 0073 if (tile.x() == m_tileX && tile.y() == m_tileY) { 0074 return m_landmass; 0075 } 0076 m_tileX = tile.x(); 0077 m_tileY = tile.y(); 0078 0079 if (!m_osmxFile.isEmpty() && m_inputType == OsmxInput) { 0080 const auto tileBox = m_tileProjection.geoCoordinates(tile); 0081 const QString bbox = QString::number(tileBox.south(GeoDataCoordinates::Degree)) 0082 + QLatin1Char(',') + QString::number(tileBox.west(GeoDataCoordinates::Degree)) 0083 + QLatin1Char(',') + QString::number(tileBox.north(GeoDataCoordinates::Degree)) 0084 + QLatin1Char(',') + QString::number(tileBox.east(GeoDataCoordinates::Degree)); 0085 0086 // TODO the following could be optimized by directly reading via OSMX API 0087 QTemporaryFile tempPbfFile(m_cacheDir + "/tmp/XXXXXX.osm.pbf"); 0088 if (!tempPbfFile.open()) { 0089 qCritical() << "Failed to open temporary file!" << tempPbfFile.errorString() << m_cacheDir; 0090 return {}; 0091 } 0092 0093 QProcess osmx; 0094 osmx.start("osmx", QStringList({ "extract" , (m_cacheDir + QLatin1Char('/') + m_osmxFile), tempPbfFile.fileName(), "--noUserData", "--bbox", bbox })); 0095 osmx.waitForFinished(5*60*1000); 0096 if (osmx.exitCode() != 0) { 0097 qWarning() << osmx.readAllStandardError(); 0098 qWarning() << "osmx failed: " << osmx.errorString() << osmx.exitStatus() << osmx.exitCode(); 0099 return {}; 0100 } 0101 0102 m_landmass = open(tempPbfFile.fileName(), m_manager); 0103 } else if (!m_osmxFile.isEmpty() && m_inputType == RawInput) { 0104 m_landmass = open(m_osmxFile, m_manager); 0105 } else { 0106 QString const filename = QString("%1/%2/%3.%4").arg(m_baseDir).arg(tile.x()).arg(tile.y()).arg("o5m"); 0107 m_landmass = open(filename, m_manager); 0108 } 0109 0110 if (m_landmass) { 0111 PeakAnalyzer::determineZoomLevel(m_landmass->placemarkList()); 0112 } 0113 return m_landmass; 0114 } 0115 0116 void TileDirectory::setInputFile(const QString &filename) 0117 { 0118 m_inputFile = filename; 0119 0120 if (m_tileType == OpenStreetMap) { 0121 QUrl url = QUrl(filename); 0122 if (url.scheme().isEmpty()) { 0123 // local file 0124 m_boundingBox = boundingBox(m_inputFile); 0125 } else { 0126 // remote file: check if already downloaded 0127 QFileInfo cacheFile = QString("%1/%2").arg(m_cacheDir).arg(url.fileName()); 0128 if (!cacheFile.exists()) { 0129 download(filename, cacheFile.absoluteFilePath()); 0130 } 0131 m_inputFile = cacheFile.absoluteFilePath(); 0132 0133 QString polyFile = QUrl(filename).fileName(); 0134 polyFile.replace("-latest.osm.pbf", ".poly"); 0135 polyFile.replace(".osm.pbf", ".poly"); 0136 polyFile.replace(".pbf", ".poly"); 0137 QString poly = QString("%1/%2").arg(url.adjusted(QUrl::RemoveFilename).toString()).arg(polyFile); 0138 QString const polyTarget = QString("%1/%2").arg(m_cacheDir).arg(polyFile); 0139 if (!QFileInfo(polyTarget).exists()) { 0140 download(poly, polyTarget); 0141 } 0142 setBoundingPolygon(polyTarget); 0143 } 0144 } 0145 } 0146 0147 GeoDataDocument* TileDirectory::clip(int zoomLevel, int tileX, int tileY) 0148 { 0149 QSharedPointer<GeoDataDocument> oldMap = m_landmass; 0150 load(zoomLevel, tileX, tileY); 0151 if (!m_clipper || oldMap != m_landmass || m_tagZoomLevel != zoomLevel) { 0152 setTagZoomLevel(zoomLevel); 0153 GeoDataDocument* input = m_tagsFilter ? m_tagsFilter->accepted() : m_landmass.data(); 0154 if (input) { 0155 m_clipper = QSharedPointer<VectorClipper>(new VectorClipper(input, m_maxZoomLevel)); 0156 } 0157 } 0158 return m_clipper ? m_clipper->clipTo(zoomLevel, tileX, tileY) : nullptr; 0159 } 0160 0161 QString TileDirectory::name() const 0162 { 0163 return QString("%1/%2/%3").arg(m_zoomLevel).arg(m_tileX).arg(m_tileY); 0164 } 0165 0166 QSharedPointer<GeoDataDocument> TileDirectory::open(const QString &filename, ParsingRunnerManager &manager) 0167 { 0168 // Timeout is set to 10 min. If the file is reaaally huge, set it to something bigger. 0169 GeoDataDocument* map = manager.openFile(filename, DocumentRole::MapDocument, 600000); 0170 if(map == nullptr) { 0171 qWarning() << "File" << filename << "couldn't be loaded."; 0172 } 0173 QSharedPointer<GeoDataDocument> result = QSharedPointer<GeoDataDocument>(map); 0174 return result; 0175 } 0176 0177 TagsFilter::Tags TileDirectory::tagsFilteredIn(int zoomLevel) const 0178 { 0179 if (m_tags.isEmpty()) { 0180 QSet<GeoDataPlacemark::GeoDataVisualCategory> categories; 0181 for (int i=GeoDataPlacemark::PlaceCity; i<GeoDataPlacemark::LastIndex; ++i) { 0182 categories << GeoDataPlacemark::GeoDataVisualCategory(i); 0183 } 0184 0185 auto const tagMap = StyleBuilder::osmTagMapping(); 0186 for (auto category: categories) { 0187 for (auto iter=tagMap.begin(), end=tagMap.end(); iter != end; ++iter) { 0188 if (iter.value() == category) { 0189 int zoomLevel = StyleBuilder::minimumZoomLevel(category); 0190 if (zoomLevel < 17) { 0191 m_tags[zoomLevel] << iter.key(); 0192 } 0193 } 0194 } 0195 } 0196 } 0197 0198 TagsFilter::Tags result; 0199 for (auto iter = m_tags.begin(), end = m_tags.end(); iter != end && iter.key() <= zoomLevel+1; ++iter) { 0200 result << iter.value(); 0201 } 0202 return result; 0203 } 0204 0205 void TileDirectory::setTagZoomLevel(int zoomLevel) 0206 { 0207 m_tagZoomLevel = zoomLevel; 0208 if (m_tileType == OpenStreetMap) { 0209 if (m_tagZoomLevel < 17) { 0210 auto const tags = tagsFilteredIn(m_tagZoomLevel); 0211 m_tagsFilter = QSharedPointer<TagsFilter>(new TagsFilter(m_landmass.data(), tags, TagsFilter::FilterRailwayService)); 0212 } else { 0213 m_tagsFilter.clear(); 0214 } 0215 } 0216 } 0217 0218 void TileDirectory::download(const QString &url, const QString &target) 0219 { 0220 m_download = QSharedPointer<Download>(new Download); 0221 m_download->target = target; 0222 m_download->reply = m_downloadManager.get(QNetworkRequest(QUrl(url))); 0223 connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), m_download.data(), SLOT(updateProgress(qint64,qint64))); 0224 connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(updateProgress())); 0225 QEventLoop loop; 0226 connect(m_download->reply, SIGNAL(finished()), &loop, SLOT(quit())); 0227 loop.exec(); 0228 cout << endl; 0229 } 0230 0231 QString TileDirectory::osmFileFor(const TileId &tileId) const 0232 { 0233 QString const outputDir = QString("%1/osm/%2/%3").arg(m_cacheDir).arg(tileId.zoomLevel()).arg(tileId.x()); 0234 return QString("%1/%2.o5m").arg(outputDir).arg(tileId.y()); 0235 } 0236 0237 void TileDirectory::printProgress(double progress, int barWidth) 0238 { 0239 int const position = barWidth * progress; 0240 cout << " [" << string(position, '=') << ">"; 0241 cout << string(barWidth-position, ' ') << "] " << std::right << setw(3) << int(progress * 100.0) << "%"; 0242 } 0243 0244 GeoDataLatLonBox TileDirectory::boundingBox() const 0245 { 0246 return m_boundingBox; 0247 } 0248 0249 void TileDirectory::setBoundingBox(const GeoDataLatLonBox &boundingBox) 0250 { 0251 m_boundingBox = boundingBox; 0252 } 0253 0254 void TileDirectory::setBoundingPolygon(const QString &file) 0255 { 0256 m_boundingPolygon.clear(); 0257 QFile input(file); 0258 QString country = "Unknown"; 0259 if ( input.open( QFile::ReadOnly ) ) { 0260 QTextStream stream( &input ); 0261 country = stream.readLine(); 0262 double lat( 0.0 ), lon( 0.0 ); 0263 GeoDataLinearRing box; 0264 while ( !stream.atEnd() ) { 0265 bool inside = true; 0266 QString line = stream.readLine().trimmed(); 0267 QStringList entries = line.split( QLatin1Char( ' ' ), QString::SkipEmptyParts ); 0268 if ( entries.size() == 1 ) { 0269 if (entries.first() == QLatin1String("END") && inside) { 0270 inside = false; 0271 if (!box.isEmpty()) { 0272 m_boundingPolygon << box; 0273 box = GeoDataLinearRing(); 0274 } 0275 } else if (entries.first() == QLatin1String("END") && !inside) { 0276 qDebug() << "END not expected here"; 0277 } else if ( entries.first().startsWith( QLatin1String( "!" ) ) ) { 0278 qDebug() << "Warning: Negative polygons not supported, skipping"; 0279 } else { 0280 //int number = entries.first().toInt(); 0281 inside = true; 0282 } 0283 } else if ( entries.size() == 2 ) { 0284 lon = entries.first().toDouble(); 0285 lat = entries.last().toDouble(); 0286 GeoDataCoordinates point( lon, lat, 0.0, GeoDataCoordinates::Degree ); 0287 box << point; 0288 } else { 0289 qDebug() << "Warning: Ignoring line in" << file 0290 << "with" << entries.size() << "fields:" << line; 0291 } 0292 } 0293 } 0294 0295 if (!m_boundingPolygon.isEmpty()) { 0296 m_boundingBox = GeoDataLatLonBox::fromLineString(m_boundingPolygon.first()); 0297 for (int i=1, n=m_boundingPolygon.size(); i<n; ++i) { 0298 m_boundingBox |= GeoDataLatLonBox::fromLineString(m_boundingPolygon[i]); 0299 } 0300 } else { 0301 m_boundingBox = boundingBox(m_inputFile); 0302 } 0303 } 0304 0305 void TileDirectory::createTiles() 0306 { 0307 if (m_tileType == OpenStreetMap) { 0308 createOsmTiles(); 0309 return; 0310 } 0311 0312 QString const landmassDir = QString("%1/land-polygons-split-4326").arg(m_cacheDir); 0313 m_inputFile = QString("%1/land_polygons.shp").arg(landmassDir); 0314 auto const landmassZip = QString("%1/%2").arg(m_cacheDir).arg(m_landmassFile); 0315 if (!QFileInfo(landmassZip).exists()) { 0316 QString const url = QString("https://osmdata.openstreetmap.de/download/%1").arg(m_landmassFile); 0317 download(url, landmassZip); 0318 } 0319 0320 if (!QFileInfo(landmassDir).exists()) { 0321 MarbleZipReader unzip(landmassZip); 0322 if (!unzip.extractAll(m_cacheDir)) { 0323 qWarning() << "Failed to extract" << landmassZip << "to" << m_cacheDir; 0324 } 0325 } 0326 0327 QSharedPointer<GeoDataDocument> map; 0328 QSharedPointer<VectorClipper> clipper; 0329 TileIterator iter(m_boundingBox, m_zoomLevel); 0330 qint64 count = 0; 0331 for(auto const &tileId: iter) { 0332 ++count; 0333 QString const outputDir = QString("%1/%2").arg(m_baseDir).arg(tileId.x()); 0334 QString const outputFile = QString("%1/%2.o5m").arg(outputDir).arg(tileId.y()); 0335 if (QFileInfo(outputFile).exists()) { 0336 continue; 0337 } 0338 0339 printProgress(count / double(iter.total())); 0340 cout << " Creating landmass cache tile " << count << "/" << iter.total() << " ("; 0341 cout << m_zoomLevel << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r'; 0342 cout.flush(); 0343 0344 QDir().mkpath(outputDir); 0345 if (!clipper) { 0346 map = open(m_inputFile, m_manager); 0347 if (!map) { 0348 qCritical() << "Failed to open " << m_inputFile << ". This can happen when Marble was compiled without shapelib (libshp), when the system has too little memory (RAM + swap need to be at least 8G), or when the download of the landmass data file failed."; 0349 } 0350 clipper = QSharedPointer<VectorClipper>(new VectorClipper(map.data(), m_zoomLevel)); 0351 } 0352 std::unique_ptr<GeoDataDocument> tile(clipper->clipTo(m_zoomLevel, tileId.x(), tileId.y())); 0353 if (!GeoDataDocumentWriter::write(outputFile, *tile)) { 0354 qWarning() << "Failed to write tile" << outputFile; 0355 } 0356 } 0357 printProgress(1.0); 0358 cout << " landmass cache tiles complete." << string(20, ' ') << endl; 0359 } 0360 0361 void TileDirectory::createOsmTiles() const 0362 { 0363 const GeoSceneMercatorTileProjection tileProjection; 0364 const QRect rect = tileProjection.tileIndexes(m_boundingBox, m_zoomLevel); 0365 TileCoordsPyramid pyramid(0, m_zoomLevel); 0366 pyramid.setBottomLevelCoords(rect); 0367 0368 qint64 const maxCount = pyramid.tilesCount(); 0369 QMap<int,QVector<TileId> > tileLevels; 0370 for (int zoomLevel=0; zoomLevel <= m_zoomLevel; ++zoomLevel) { 0371 QRect const rect = pyramid.coords(zoomLevel); 0372 if (zoomLevel < m_zoomLevel && rect.width()*rect.height() < 2) { 0373 continue; 0374 } 0375 0376 TileIterator iter(m_boundingBox, zoomLevel); 0377 for(auto const &tileId: iter) { 0378 tileLevels[zoomLevel] << TileId(0, zoomLevel, tileId.x(), tileId.y()); 0379 } 0380 } 0381 0382 bool hasAllTiles = true; 0383 for (auto const &tileId: tileLevels[m_zoomLevel]) { 0384 auto const outputFile = osmFileFor(tileId); 0385 if (!QFileInfo(outputFile).exists()) { 0386 hasAllTiles = false; 0387 break; 0388 } 0389 } 0390 0391 bool first = true; 0392 if (!hasAllTiles) { 0393 qint64 count = 0; 0394 for (auto const &tiles: tileLevels) { 0395 for (auto const &tileId: tiles) { 0396 ++count; 0397 QString const inputFile = first ? m_inputFile : QString("%1/osm/%2/%3/%4.o5m"). 0398 arg(m_cacheDir).arg(tileId.zoomLevel()-1).arg(tileId.x()>>1).arg(tileId.y()>>1); 0399 QString const outputFile = osmFileFor(tileId); 0400 if (QFileInfo(outputFile).exists()) { 0401 continue; 0402 } 0403 0404 printProgress(count / double(maxCount)); 0405 cout << " Creating osm cache tile " << count << "/" << maxCount << " ("; 0406 cout << tileId.zoomLevel() << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r'; 0407 cout.flush(); 0408 0409 QDir().mkpath(QFileInfo(outputFile).absolutePath()); 0410 QString const output = QString("-o=%1").arg(outputFile); 0411 0412 const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tileId.zoomLevel(), tileId.x(), tileId.y()); 0413 0414 double const minLon = tileBoundary.west(GeoDataCoordinates::Degree); 0415 double const maxLon = tileBoundary.east(GeoDataCoordinates::Degree); 0416 double const maxLat = tileBoundary.north(GeoDataCoordinates::Degree); 0417 double const minLat = tileBoundary.south(GeoDataCoordinates::Degree); 0418 QString const bbox = QString("-b=%1,%2,%3,%4").arg(minLon).arg(minLat).arg(maxLon).arg(maxLat); 0419 QProcess osmconvert; 0420 osmconvert.start("osmconvert", QStringList() << "--drop-author" << "--drop-version" 0421 << "--complete-ways" << "--complex-ways" << bbox << output << inputFile); 0422 osmconvert.waitForFinished(10*60*1000); 0423 if (osmconvert.exitCode() != 0) { 0424 qWarning() << osmconvert.readAllStandardError(); 0425 qWarning() << "osmconvert failed: " << osmconvert.errorString(); 0426 } 0427 } 0428 first = false; 0429 } 0430 } 0431 0432 tileLevels.remove(m_zoomLevel); 0433 for (auto const &tiles: tileLevels) { 0434 for (auto const &tileId: tiles) { 0435 QFile::remove(osmFileFor(tileId)); 0436 } 0437 } 0438 0439 printProgress(1.0); 0440 cout << " osm cache tiles complete." << string(20, ' ') << endl; 0441 0442 } 0443 0444 int TileDirectory::innerNodes(const TileId &tile) const 0445 { 0446 const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tile.zoomLevel(), tile.x(), tile.y()); 0447 0448 double const west = tileBoundary.west(); 0449 double const east = tileBoundary.east(); 0450 double const north = tileBoundary.north(); 0451 double const south = tileBoundary.south(); 0452 QVector<GeoDataCoordinates> bounds; 0453 bounds << GeoDataCoordinates(west, north); 0454 bounds << GeoDataCoordinates(east, north); 0455 bounds << GeoDataCoordinates(east, south); 0456 bounds << GeoDataCoordinates(west, south); 0457 0458 int innerNodes = 0; 0459 if (m_boundingPolygon.isEmpty()) { 0460 for(auto const &coordinate: bounds) { 0461 if (m_boundingBox.contains(coordinate)) { 0462 ++innerNodes; 0463 } 0464 } 0465 return innerNodes; 0466 } 0467 0468 for(auto const &coordinate: bounds) { 0469 for(auto const &ring: m_boundingPolygon) { 0470 if (ring.contains(coordinate)) { 0471 ++innerNodes; 0472 } 0473 } 0474 } 0475 return innerNodes; 0476 } 0477 0478 void TileDirectory::updateProgress() 0479 { 0480 double const progress = m_download->total > 0 ? m_download->received / double(m_download->total) : 0.0; 0481 printProgress(progress); 0482 0483 cout << " "; 0484 cout << std::fixed << std::setprecision(1) << m_download->received / 1000000.0 << '/'; 0485 cout << std::fixed << std::setprecision(1) << m_download->total / 1000000.0 << " MB"; 0486 0487 cout << " Downloading " << m_download->reply->url().fileName().toStdString(); 0488 0489 cout << string(20, ' ') << '\r'; 0490 cout.flush(); 0491 } 0492 0493 void TileDirectory::handleFinishedDownload(const QString &filename, const QString &id) 0494 { 0495 qDebug() << "File " << filename << "(" << id << ") has been downloaded."; 0496 } 0497 0498 GeoDataLatLonBox TileDirectory::boundingBox(const QString &filename) const 0499 { 0500 QProcess osmconvert; 0501 osmconvert.start("osmconvert", QStringList() << "--out-statistics" << filename); 0502 osmconvert.waitForFinished(10*60*1000); 0503 QStringList const output = QString(osmconvert.readAllStandardOutput()).split('\n'); 0504 GeoDataLatLonBox boundingBox; 0505 for(QString const &line: output) { 0506 if (line.startsWith("lon min:")) { 0507 boundingBox.setWest(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0508 } else if (line.startsWith("lon max")) { 0509 boundingBox.setEast(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0510 } else if (line.startsWith("lat min:")) { 0511 boundingBox.setSouth(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0512 } else if (line.startsWith("lat max:")) { 0513 boundingBox.setNorth(line.mid(8).toDouble(), GeoDataCoordinates::Degree); 0514 } 0515 } 0516 return boundingBox; 0517 } 0518 0519 void Download::updateProgress(qint64 received_, qint64 total_) 0520 { 0521 received = received_; 0522 total = total_; 0523 0524 QString const tempFile = QString("%1.download").arg(target); 0525 if (!m_file.isOpen()) { 0526 m_file.setFileName(tempFile); 0527 m_file.open(QFile::WriteOnly); 0528 } 0529 m_file.write(reply->readAll()); 0530 0531 if (reply->isFinished()) { 0532 m_file.flush(); 0533 m_file.close(); 0534 QFile::rename(tempFile, target); 0535 } 0536 } 0537 0538 } 0539 0540 #include "moc_TileDirectory.cpp"