tools/vectorosm-tilecreator/TileDirectory.cpp

0001 // SPDX-License-Identifier: LGPL-2.1-or-later
0002 //
0003 // SPDX-FileCopyrightText: 2016 Dennis Nienhüser <nienhueser@kde.org>
0004 //
0005
0006 #include "TileDirectory.h"
0007 #include "TileIterator.h"
0008 #include <GeoDataDocumentWriter.h>
0009 #include "MarbleZipReader.h"
0010 #include <GeoDataLatLonAltBox.h>
0011 #include "PeakAnalyzer.h"
0012 #include "TileCoordsPyramid.h"
0013 #include "StyleBuilder.h"
0014
0015 #include <QFileInfo>
0016 #include <QDebug>
0017 #include <QProcess>
0018 #include <QDir>
0019 #include <QUrl>
0020 #include <QNetworkRequest>
0021 #include <QNetworkReply>
0022 #include <QTemporaryFile>
0023 #include <QThread>
0024
0025 #include <iostream>
0026 #include <iomanip>
0027
0028 using namespace std;
0029
0030 namespace Marble {
0031
0032 QMap<int, TagsFilter::Tags> TileDirectory::m_tags;
0033
0034 TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, int maxZoomLevel) :
0035     m_cacheDir(cacheDir),
0036     m_manager(manager),
0037     m_tileType(tileType),
0038     m_landmassFile("land-polygons-split-4326.zip"),
0039     m_maxZoomLevel(maxZoomLevel)
0040 {
0041     if (m_tileType == Landmass) {
0042         m_zoomLevel = 7;
0043         m_baseDir = QString("%1/landmass/%2").arg(cacheDir).arg(m_zoomLevel);
0044     } else {
0045         m_zoomLevel = 10;
0046         m_baseDir = QString("%1/osm/%2").arg(cacheDir).arg(m_zoomLevel);
0047     }
0048     QDir().mkpath(m_baseDir);
0049 }
0050
0051 TileDirectory::TileDirectory(const QString &cacheDir, const QString &osmxFile, ParsingRunnerManager &manager, int maxZoomLevel, int loadZoomLevel, InputType inputType) :
0052     m_cacheDir(cacheDir),
0053     m_osmxFile(osmxFile),
0054     m_manager(manager),
0055     m_zoomLevel(loadZoomLevel),
0056     m_tileType(OpenStreetMap),
0057     m_inputType(inputType),
0058     m_maxZoomLevel(maxZoomLevel)
0059 {
0060 }
0061
0062 TileId TileDirectory::tileFor(int zoomLevel, int tileX, int tileY) const
0063 {
0064     int const zoomDiff = zoomLevel - m_zoomLevel;
0065     int const x = tileX >> zoomDiff;
0066     int const y = tileY >> zoomDiff;
0067     return TileId(QString(), m_zoomLevel, x, y);
0068 }
0069
0070 QSharedPointer<GeoDataDocument> TileDirectory::load(int zoomLevel, int tileX, int tileY)
0071 {
0072     auto const tile = tileFor(zoomLevel, tileX, tileY);
0073     if (tile.x() == m_tileX && tile.y() == m_tileY) {
0074         return m_landmass;
0075     }
0076     m_tileX = tile.x();
0077     m_tileY = tile.y();
0078
0079     if (!m_osmxFile.isEmpty() && m_inputType == OsmxInput) {
0080         const auto tileBox = m_tileProjection.geoCoordinates(tile);
0081         const QString bbox = QString::number(tileBox.south(GeoDataCoordinates::Degree))
0082             + QLatin1Char(',') + QString::number(tileBox.west(GeoDataCoordinates::Degree))
0083             + QLatin1Char(',') + QString::number(tileBox.north(GeoDataCoordinates::Degree))
0084             + QLatin1Char(',') + QString::number(tileBox.east(GeoDataCoordinates::Degree));
0085
0086         // TODO the following could be optimized by directly reading via OSMX API
0087         QTemporaryFile tempPbfFile(m_cacheDir + "/tmp/XXXXXX.osm.pbf");
0088         if (!tempPbfFile.open()) {
0089             qCritical() << "Failed to open temporary file!" << tempPbfFile.errorString() << m_cacheDir;
0090             return {};
0091         }
0092
0093         QProcess osmx;
0094         osmx.start("osmx", QStringList({ "extract" , (m_cacheDir + QLatin1Char('/') + m_osmxFile), tempPbfFile.fileName(), "--noUserData", "--bbox", bbox }));
0095         osmx.waitForFinished(5*60*1000);
0096         if (osmx.exitCode() != 0) {
0097             qWarning() << osmx.readAllStandardError();
0098             qWarning() << "osmx failed: " << osmx.errorString() << osmx.exitStatus() << osmx.exitCode();
0099             return {};
0100         }
0101
0102         m_landmass = open(tempPbfFile.fileName(), m_manager);
0103     } else if (!m_osmxFile.isEmpty() && m_inputType == RawInput) {
0104         m_landmass = open(m_osmxFile, m_manager);
0105     } else {
0106         QString const filename = QString("%1/%2/%3.%4").arg(m_baseDir).arg(tile.x()).arg(tile.y()).arg("o5m");
0107         m_landmass = open(filename, m_manager);
0108     }
0109
0110     if (m_landmass) {
0111         PeakAnalyzer::determineZoomLevel(m_landmass->placemarkList());
0112     }
0113     return m_landmass;
0114 }
0115
0116 void TileDirectory::setInputFile(const QString &filename)
0117 {
0118     m_inputFile = filename;
0119
0120     if (m_tileType == OpenStreetMap) {
0121         QUrl url = QUrl(filename);
0122         if (url.scheme().isEmpty()) {
0123             // local file
0124             m_boundingBox = boundingBox(m_inputFile);
0125         } else {
0126             // remote file: check if already downloaded
0127             QFileInfo cacheFile = QString("%1/%2").arg(m_cacheDir).arg(url.fileName());
0128             if (!cacheFile.exists()) {
0129                 download(filename, cacheFile.absoluteFilePath());
0130             }
0131             m_inputFile = cacheFile.absoluteFilePath();
0132
0133             QString polyFile = QUrl(filename).fileName();
0134             polyFile.replace("-latest.osm.pbf", ".poly");
0135             polyFile.replace(".osm.pbf", ".poly");
0136             polyFile.replace(".pbf", ".poly");
0137             QString poly = QString("%1/%2").arg(url.adjusted(QUrl::RemoveFilename).toString()).arg(polyFile);
0138             QString const polyTarget = QString("%1/%2").arg(m_cacheDir).arg(polyFile);
0139             if (!QFileInfo(polyTarget).exists()) {
0140                 download(poly, polyTarget);
0141             }
0142             setBoundingPolygon(polyTarget);
0143         }
0144     }
0145 }
0146
0147 GeoDataDocument* TileDirectory::clip(int zoomLevel, int tileX, int tileY)
0148 {
0149     QSharedPointer<GeoDataDocument> oldMap = m_landmass;
0150     load(zoomLevel, tileX, tileY);
0151     if (!m_clipper || oldMap != m_landmass || m_tagZoomLevel != zoomLevel) {
0152         setTagZoomLevel(zoomLevel);
0153         GeoDataDocument* input = m_tagsFilter ? m_tagsFilter->accepted() : m_landmass.data();
0154         if (input) {
0155             m_clipper = QSharedPointer<VectorClipper>(new VectorClipper(input, m_maxZoomLevel));
0156         }
0157     }
0158     return m_clipper ? m_clipper->clipTo(zoomLevel, tileX, tileY) : nullptr;
0159 }
0160
0161 QString TileDirectory::name() const
0162 {
0163     return QString("%1/%2/%3").arg(m_zoomLevel).arg(m_tileX).arg(m_tileY);
0164 }
0165
0166 QSharedPointer<GeoDataDocument> TileDirectory::open(const QString &filename, ParsingRunnerManager &manager)
0167 {
0168     // Timeout is set to 10 min. If the file is reaaally huge, set it to something bigger.
0169     GeoDataDocument* map = manager.openFile(filename, DocumentRole::MapDocument, 600000);
0170     if(map == nullptr) {
0171         qWarning() << "File" << filename << "couldn't be loaded.";
0172     }
0173     QSharedPointer<GeoDataDocument> result = QSharedPointer<GeoDataDocument>(map);
0174     return result;
0175 }
0176
0177 TagsFilter::Tags TileDirectory::tagsFilteredIn(int zoomLevel) const
0178 {
0179     if (m_tags.isEmpty()) {
0180         QSet<GeoDataPlacemark::GeoDataVisualCategory> categories;
0181         for (int i=GeoDataPlacemark::PlaceCity; i<GeoDataPlacemark::LastIndex; ++i) {
0182             categories << GeoDataPlacemark::GeoDataVisualCategory(i);
0183         }
0184
0185         auto const tagMap = StyleBuilder::osmTagMapping();
0186         for (auto category: categories) {
0187             for (auto iter=tagMap.begin(), end=tagMap.end(); iter != end; ++iter) {
0188                 if (iter.value() == category) {
0189                     int zoomLevel = StyleBuilder::minimumZoomLevel(category);
0190                     if (zoomLevel < 17) {
0191                         m_tags[zoomLevel] << iter.key();
0192                     }
0193                 }
0194             }
0195         }
0196     }
0197
0198     TagsFilter::Tags result;
0199     for (auto iter = m_tags.begin(), end = m_tags.end(); iter != end && iter.key() <= zoomLevel+1; ++iter) {
0200         result << iter.value();
0201     }
0202     return result;
0203 }
0204
0205 void TileDirectory::setTagZoomLevel(int zoomLevel)
0206 {
0207     m_tagZoomLevel = zoomLevel;
0208     if (m_tileType == OpenStreetMap) {
0209         if (m_tagZoomLevel < 17) {
0210             auto const tags = tagsFilteredIn(m_tagZoomLevel);
0211             m_tagsFilter = QSharedPointer<TagsFilter>(new TagsFilter(m_landmass.data(), tags, TagsFilter::FilterRailwayService));
0212         } else {
0213             m_tagsFilter.clear();
0214         }
0215     }
0216 }
0217
0218 void TileDirectory::download(const QString &url, const QString &target)
0219 {
0220     m_download = QSharedPointer<Download>(new Download);
0221     m_download->target = target;
0222     m_download->reply = m_downloadManager.get(QNetworkRequest(QUrl(url)));
0223     connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), m_download.data(), SLOT(updateProgress(qint64,qint64)));
0224     connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(updateProgress()));
0225     QEventLoop loop;
0226     connect(m_download->reply, SIGNAL(finished()), &loop, SLOT(quit()));
0227     loop.exec();
0228     cout << endl;
0229 }
0230
0231 QString TileDirectory::osmFileFor(const TileId &tileId) const
0232 {
0233     QString const outputDir = QString("%1/osm/%2/%3").arg(m_cacheDir).arg(tileId.zoomLevel()).arg(tileId.x());
0234     return QString("%1/%2.o5m").arg(outputDir).arg(tileId.y());
0235 }
0236
0237 void TileDirectory::printProgress(double progress, int barWidth)
0238 {
0239     int const position = barWidth * progress;
0240     cout << " [" << string(position, '=') << ">";
0241     cout << string(barWidth-position, ' ') << "]  " << std::right << setw(3) << int(progress * 100.0) << "%";
0242 }
0243
0244 GeoDataLatLonBox TileDirectory::boundingBox() const
0245 {
0246     return m_boundingBox;
0247 }
0248
0249 void TileDirectory::setBoundingBox(const GeoDataLatLonBox &boundingBox)
0250 {
0251     m_boundingBox = boundingBox;
0252 }
0253
0254 void TileDirectory::setBoundingPolygon(const QString &file)
0255 {
0256     m_boundingPolygon.clear();
0257     QFile input(file);
0258     QString country = "Unknown";
0259     if ( input.open( QFile::ReadOnly ) ) {
0260         QTextStream stream( &input );
0261         country = stream.readLine();
0262         double lat( 0.0 ), lon( 0.0 );
0263         GeoDataLinearRing box;
0264         while ( !stream.atEnd() ) {
0265             bool inside = true;
0266             QString line = stream.readLine().trimmed();
0267             QStringList entries = line.split( QLatin1Char( ' ' ), QString::SkipEmptyParts );
0268             if ( entries.size() == 1 ) {
0269                 if (entries.first() == QLatin1String("END") && inside) {
0270                     inside = false;
0271                     if (!box.isEmpty()) {
0272                         m_boundingPolygon << box;
0273                         box = GeoDataLinearRing();
0274                     }
0275                 } else if (entries.first() == QLatin1String("END") && !inside) {
0276                     qDebug() << "END not expected here";
0277                 } else if ( entries.first().startsWith( QLatin1String( "!" ) ) ) {
0278                     qDebug() << "Warning: Negative polygons not supported, skipping";
0279                 } else {
0280                     //int number = entries.first().toInt();
0281                     inside = true;
0282                 }
0283             } else if ( entries.size() == 2 ) {
0284                 lon = entries.first().toDouble();
0285                 lat = entries.last().toDouble();
0286                 GeoDataCoordinates point( lon, lat, 0.0, GeoDataCoordinates::Degree );
0287                 box << point;
0288             } else {
0289                 qDebug() << "Warning: Ignoring line in" << file
0290                          <<  "with" << entries.size() << "fields:" << line;
0291             }
0292         }
0293     }
0294
0295     if (!m_boundingPolygon.isEmpty()) {
0296         m_boundingBox = GeoDataLatLonBox::fromLineString(m_boundingPolygon.first());
0297         for (int i=1, n=m_boundingPolygon.size(); i<n; ++i) {
0298             m_boundingBox |= GeoDataLatLonBox::fromLineString(m_boundingPolygon[i]);
0299         }
0300     } else {
0301         m_boundingBox = boundingBox(m_inputFile);
0302     }
0303 }
0304
0305 void TileDirectory::createTiles()
0306 {
0307     if (m_tileType == OpenStreetMap) {
0308         createOsmTiles();
0309         return;
0310     }
0311
0312     QString const landmassDir = QString("%1/land-polygons-split-4326").arg(m_cacheDir);
0313     m_inputFile = QString("%1/land_polygons.shp").arg(landmassDir);
0314     auto const landmassZip = QString("%1/%2").arg(m_cacheDir).arg(m_landmassFile);
0315     if (!QFileInfo(landmassZip).exists()) {
0316         QString const url = QString("https://osmdata.openstreetmap.de/download/%1").arg(m_landmassFile);
0317         download(url, landmassZip);
0318     }
0319
0320     if (!QFileInfo(landmassDir).exists()) {
0321         MarbleZipReader unzip(landmassZip);
0322         if (!unzip.extractAll(m_cacheDir)) {
0323             qWarning() << "Failed to extract" << landmassZip << "to" << m_cacheDir;
0324         }
0325     }
0326
0327     QSharedPointer<GeoDataDocument> map;
0328     QSharedPointer<VectorClipper> clipper;
0329     TileIterator iter(m_boundingBox, m_zoomLevel);
0330     qint64 count = 0;
0331     for(auto const &tileId: iter) {
0332         ++count;
0333         QString const outputDir = QString("%1/%2").arg(m_baseDir).arg(tileId.x());
0334         QString const outputFile = QString("%1/%2.o5m").arg(outputDir).arg(tileId.y());
0335         if (QFileInfo(outputFile).exists()) {
0336             continue;
0337         }
0338
0339         printProgress(count / double(iter.total()));
0340         cout << " Creating landmass cache tile " << count << "/" << iter.total() << " (";
0341         cout << m_zoomLevel << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r';
0342         cout.flush();
0343
0344         QDir().mkpath(outputDir);
0345         if (!clipper) {
0346             map = open(m_inputFile, m_manager);
0347             if (!map) {
0348                 qCritical() << "Failed to open " << m_inputFile << ". This can happen when Marble was compiled without shapelib (libshp), when the system has too little memory (RAM + swap need to be at least 8G), or when the download of the landmass data file failed.";
0349             }
0350             clipper = QSharedPointer<VectorClipper>(new VectorClipper(map.data(), m_zoomLevel));
0351         }
0352         std::unique_ptr<GeoDataDocument> tile(clipper->clipTo(m_zoomLevel, tileId.x(), tileId.y()));
0353         if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
0354             qWarning() << "Failed to write tile" << outputFile;
0355         }
0356     }
0357     printProgress(1.0);
0358     cout << "  landmass cache tiles complete." << string(20, ' ') << endl;
0359 }
0360
0361 void TileDirectory::createOsmTiles() const
0362 {
0363     const GeoSceneMercatorTileProjection tileProjection;
0364     const QRect rect = tileProjection.tileIndexes(m_boundingBox, m_zoomLevel);
0365     TileCoordsPyramid pyramid(0, m_zoomLevel);
0366     pyramid.setBottomLevelCoords(rect);
0367
0368     qint64 const maxCount = pyramid.tilesCount();
0369     QMap<int,QVector<TileId> > tileLevels;
0370     for (int zoomLevel=0; zoomLevel <= m_zoomLevel; ++zoomLevel) {
0371         QRect const rect = pyramid.coords(zoomLevel);
0372         if (zoomLevel < m_zoomLevel && rect.width()*rect.height() < 2) {
0373             continue;
0374         }
0375
0376         TileIterator iter(m_boundingBox, zoomLevel);
0377         for(auto const &tileId: iter) {
0378             tileLevels[zoomLevel] << TileId(0, zoomLevel, tileId.x(), tileId.y());
0379         }
0380     }
0381
0382     bool hasAllTiles = true;
0383     for (auto const &tileId: tileLevels[m_zoomLevel]) {
0384         auto const outputFile = osmFileFor(tileId);
0385         if (!QFileInfo(outputFile).exists()) {
0386             hasAllTiles = false;
0387             break;
0388         }
0389     }
0390
0391     bool first = true;
0392     if (!hasAllTiles) {
0393         qint64 count = 0;
0394         for (auto const &tiles: tileLevels) {
0395             for (auto const &tileId: tiles) {
0396                 ++count;
0397                 QString const inputFile = first ? m_inputFile : QString("%1/osm/%2/%3/%4.o5m").
0398                                                   arg(m_cacheDir).arg(tileId.zoomLevel()-1).arg(tileId.x()>>1).arg(tileId.y()>>1);
0399                 QString const outputFile = osmFileFor(tileId);
0400                 if (QFileInfo(outputFile).exists()) {
0401                     continue;
0402                 }
0403
0404                 printProgress(count / double(maxCount));
0405                 cout << " Creating osm cache tile " << count << "/" << maxCount << " (";
0406                 cout << tileId.zoomLevel() << "/" << tileId.x() << "/" << tileId.y() << ')' << string(20, ' ') << '\r';
0407                 cout.flush();
0408
0409                 QDir().mkpath(QFileInfo(outputFile).absolutePath());
0410                 QString const output = QString("-o=%1").arg(outputFile);
0411
0412                 const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tileId.zoomLevel(), tileId.x(), tileId.y());
0413
0414                 double const minLon = tileBoundary.west(GeoDataCoordinates::Degree);
0415                 double const maxLon = tileBoundary.east(GeoDataCoordinates::Degree);
0416                 double const maxLat = tileBoundary.north(GeoDataCoordinates::Degree);
0417                 double const minLat = tileBoundary.south(GeoDataCoordinates::Degree);
0418                 QString const bbox = QString("-b=%1,%2,%3,%4").arg(minLon).arg(minLat).arg(maxLon).arg(maxLat);
0419                 QProcess osmconvert;
0420                 osmconvert.start("osmconvert", QStringList() << "--drop-author" << "--drop-version"
0421                                  << "--complete-ways" << "--complex-ways" << bbox << output << inputFile);
0422                 osmconvert.waitForFinished(10*60*1000);
0423                 if (osmconvert.exitCode() != 0) {
0424                     qWarning() << osmconvert.readAllStandardError();
0425                     qWarning() << "osmconvert failed: " << osmconvert.errorString();
0426                 }
0427             }
0428             first = false;
0429         }
0430     }
0431
0432     tileLevels.remove(m_zoomLevel);
0433     for (auto const &tiles: tileLevels) {
0434         for (auto const &tileId: tiles) {
0435             QFile::remove(osmFileFor(tileId));
0436         }
0437     }
0438
0439     printProgress(1.0);
0440     cout << "  osm cache tiles complete." << string(20, ' ') << endl;
0441
0442 }
0443
0444 int TileDirectory::innerNodes(const TileId &tile) const
0445 {
0446     const GeoDataLatLonBox tileBoundary = m_tileProjection.geoCoordinates(tile.zoomLevel(), tile.x(), tile.y());
0447
0448     double const west = tileBoundary.west();
0449     double const east = tileBoundary.east();
0450     double const north = tileBoundary.north();
0451     double const south = tileBoundary.south();
0452     QVector<GeoDataCoordinates> bounds;
0453     bounds << GeoDataCoordinates(west, north);
0454     bounds << GeoDataCoordinates(east, north);
0455     bounds << GeoDataCoordinates(east, south);
0456     bounds << GeoDataCoordinates(west, south);
0457
0458     int innerNodes = 0;
0459     if (m_boundingPolygon.isEmpty()) {
0460         for(auto const &coordinate: bounds) {
0461             if (m_boundingBox.contains(coordinate)) {
0462                 ++innerNodes;
0463             }
0464         }
0465         return innerNodes;
0466     }
0467
0468     for(auto const &coordinate: bounds) {
0469         for(auto const &ring: m_boundingPolygon) {
0470             if (ring.contains(coordinate)) {
0471                 ++innerNodes;
0472             }
0473         }
0474     }
0475     return innerNodes;
0476 }
0477
0478 void TileDirectory::updateProgress()
0479 {
0480     double const progress = m_download->total > 0 ? m_download->received / double(m_download->total) : 0.0;
0481     printProgress(progress);
0482
0483     cout << "  ";
0484     cout << std::fixed << std::setprecision(1) << m_download->received / 1000000.0 << '/';
0485     cout << std::fixed << std::setprecision(1) << m_download->total / 1000000.0 << " MB";
0486
0487     cout << "  Downloading " << m_download->reply->url().fileName().toStdString();
0488
0489     cout << string(20, ' ') << '\r';
0490     cout.flush();
0491 }
0492
0493 void TileDirectory::handleFinishedDownload(const QString &filename, const QString &id)
0494 {
0495     qDebug() << "File " << filename << "(" << id << ") has been downloaded.";
0496 }
0497
0498 GeoDataLatLonBox TileDirectory::boundingBox(const QString &filename) const
0499 {
0500     QProcess osmconvert;
0501     osmconvert.start("osmconvert", QStringList() << "--out-statistics" << filename);
0502     osmconvert.waitForFinished(10*60*1000);
0503     QStringList const output = QString(osmconvert.readAllStandardOutput()).split('\n');
0504     GeoDataLatLonBox boundingBox;
0505     for(QString const &line: output) {
0506         if (line.startsWith("lon min:")) {
0507             boundingBox.setWest(line.mid(8).toDouble(), GeoDataCoordinates::Degree);
0508         } else if (line.startsWith("lon max")) {
0509             boundingBox.setEast(line.mid(8).toDouble(), GeoDataCoordinates::Degree);
0510         } else if (line.startsWith("lat min:")) {
0511             boundingBox.setSouth(line.mid(8).toDouble(), GeoDataCoordinates::Degree);
0512         } else if (line.startsWith("lat max:")) {
0513             boundingBox.setNorth(line.mid(8).toDouble(), GeoDataCoordinates::Degree);
0514         }
0515     }
0516     return boundingBox;
0517 }
0518
0519 void Download::updateProgress(qint64 received_, qint64 total_)
0520 {
0521     received = received_;
0522     total = total_;
0523
0524     QString const tempFile = QString("%1.download").arg(target);
0525     if (!m_file.isOpen()) {
0526         m_file.setFileName(tempFile);
0527         m_file.open(QFile::WriteOnly);
0528     }
0529     m_file.write(reply->readAll());
0530
0531     if (reply->isFinished()) {
0532         m_file.flush();
0533         m_file.close();
0534         QFile::rename(tempFile, target);
0535     }
0536 }
0537
0538 }
0539
0540 #include "moc_TileDirectory.cpp"