tools/vectorosm-tilecreator/vectorosm-tilecreator.cpp

0001 // SPDX-License-Identifier: LGPL-2.1-or-later
0002 //
0003 // SPDX-FileCopyrightText: 2016 David Kolozsvari <freedawson@gmail.com>
0004 // SPDX-FileCopyrightText: 2016 Dennis Nienhüser <nienhueser@kde.org>
0005 //
0006
0007 #include "GeoDataDocumentWriter.h"
0008 #include "MarbleModel.h"
0009 #include "ParsingRunnerManager.h"
0010 #include "GeoDataGeometry.h"
0011 #include "GeoDataPlacemark.h"
0012 #include "GeoDataPolygon.h"
0013 #include "GeoDataLatLonAltBox.h"
0014 #include "TileId.h"
0015 #include "MarbleDirs.h"
0016 #ifdef STATIC_BUILD
0017 #include "src/plugins/runner/osm/translators/O5mWriter.h"
0018 #endif
0019
0020 #include <QApplication>
0021 #include <QCommandLineParser>
0022 #include <QDebug>
0023 #include <QFileInfo>
0024 #include <QDir>
0025 #include <QString>
0026 #include <QElapsedTimer>
0027 #include <QSharedPointer>
0028 #include <QUrl>
0029 #include <QBuffer>
0030
0031 #include <QMessageLogContext>
0032 #include <QProcess>
0033
0034 #include "VectorClipper.h"
0035 #include "NodeReducer.h"
0036 #include "WayConcatenator.h"
0037 #include "TileIterator.h"
0038 #include "TileDirectory.h"
0039 #include "MbTileWriter.h"
0040 #include "SpellChecker.h"
0041
0042 #ifdef STATIC_BUILD
0043 #include <QtPlugin>
0044 Q_IMPORT_PLUGIN(OsmPlugin)
0045 Q_IMPORT_PLUGIN(ShpPlugin)
0046 #endif
0047
0048 #include <iostream>
0049
0050 using namespace Marble;
0051
0052 GeoDataDocument* mergeDocuments(GeoDataDocument* map1, GeoDataDocument* map2)
0053 {
0054     GeoDataDocument* mergedMap = new GeoDataDocument(*map1);
0055
0056     OsmPlacemarkData marbleLand;
0057     marbleLand.addTag("marble_land","landmass");
0058     for (auto placemark: map2->placemarkList()) {
0059         GeoDataPlacemark* land = new GeoDataPlacemark(*placemark);
0060         if (geodata_cast<GeoDataPolygon>(land->geometry())) {
0061             land->setOsmData(marbleLand);
0062         }
0063         mergedMap->append(land);
0064     }
0065
0066     return mergedMap;
0067 }
0068
0069 QString tileFileName(const QCommandLineParser &parser, int x, int y, int zoomLevel)
0070 {
0071     QString const extension = parser.value("extension");
0072     QString const output = parser.isSet("development") ? QString("%1/maps/earth/vectorosm-dev").arg(MarbleDirs::localPath()) : parser.value("output");
0073     QString const outputDir = QString("%1/%2/%3").arg(output).arg(zoomLevel).arg(x);
0074     QString const outputFile = QString("%1/%2.%3").arg(outputDir).arg(y).arg(extension);
0075     return outputFile;
0076 }
0077
0078 void writeBoundaryTile(GeoDataDocument* tile, const QString &region, const QCommandLineParser &parser, int x, int y, int zoomLevel)
0079 {
0080     QString const extension = parser.value("extension");
0081     QString const outputDir = QString("%1/boundaries/%2/%3/%4").arg(parser.value("cache-directory")).arg(region).arg(zoomLevel).arg(x);
0082     QString const outputFile = QString("%1/%2.%3").arg(outputDir).arg(y).arg(extension);
0083     QDir().mkpath(outputDir);
0084     GeoDataDocumentWriter::write(outputFile, *tile);
0085 }
0086
0087 QSharedPointer<GeoDataDocument> mergeBoundaryTiles(const QSharedPointer<GeoDataDocument> &background, ParsingRunnerManager &manager, const QCommandLineParser &parser, int x, int y, int zoomLevel)
0088 {
0089     GeoDataDocument* mergedMap = new GeoDataDocument;
0090     OsmPlacemarkData marbleLand;
0091     marbleLand.addTag("marble_land","landmass");
0092     for (auto placemark: background->placemarkList()) {
0093         GeoDataPlacemark* land = new GeoDataPlacemark(*placemark);
0094         if (geodata_cast<GeoDataPolygon>(land->geometry())) {
0095             land->setOsmData(marbleLand);
0096         }
0097         mergedMap->append(land);
0098     }
0099
0100     QString const extension = parser.value("extension");
0101     QString const boundaryDir = QString("%1/boundaries").arg(parser.value("cache-directory"));
0102     for(auto const &dir: QDir(boundaryDir).entryList(QDir::Dirs | QDir::NoDotAndDotDot)) {
0103         QString const file = QString("%1/%2/%3/%4/%5.%6").arg(boundaryDir).arg(dir).arg(zoomLevel).arg(x).arg(y).arg(extension);
0104         if (QFileInfo(file).exists()) {
0105             auto tile = TileDirectory::open(file, manager);
0106             if (tile) {
0107                 for (auto placemark: tile->placemarkList()) {
0108                     mergedMap->append(placemark->clone());
0109                 }
0110             }
0111         }
0112     }
0113
0114     return QSharedPointer<GeoDataDocument>(mergedMap);
0115 }
0116
0117 bool writeTile(GeoDataDocument* tile, const QString &outputFile)
0118 {
0119     QDir().mkpath(QFileInfo(outputFile).path());
0120     if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
0121         qWarning() << "Could not write the file " << outputFile;
0122         return false;
0123     }
0124     return true;
0125 }
0126
0127 int main(int argc, char *argv[])
0128 {
0129     QCoreApplication app(argc, argv);
0130
0131     QCoreApplication::setApplicationName("marble-vectorosm-tilecreator");
0132     QCoreApplication::setApplicationVersion("0.1");
0133
0134     QCommandLineParser parser;
0135     parser.setApplicationDescription("A tool for Marble, which is used to reduce the details of osm maps.");
0136     parser.addHelpOption();
0137     parser.addVersionOption();
0138     parser.addPositionalArgument("input", "The input .osm or .shp file.");
0139
0140     parser.addOptions({
0141                           {{"t", "osmconvert"}, "Tile data using osmconvert."},
0142                           {"conflict-resolution", "How to deal with existing tiles: overwrite, skip or merge", "mode", "overwrite"},
0143                           {{"c", "cache-directory"}, "Directory for temporary data.", "cache", "cache"},
0144                           {{"m", "mbtile"}, "Store tiles at level 15 onwards in a mbtile database.", "mbtile"},
0145                           {{"s", "spellcheck"}, "Use this geonames.org cities file for spell-checking city names", "spellcheck"},
0146                           {"verbose", "Increase amount of shell output information"},
0147                           {"boundaries", "Write boundary tiles (implied by conflict-resolution=merge)"},
0148                           {{"d", "development"}, "Use local development vector osm map theme as output storage"},
0149                           {{"z", "zoom-level"}, "Zoom level according to which OSM information has to be processed.", "levels", "11,13,15,17"},
0150                           {{"o", "output"}, "Output file or directory", "output", QString("%1/maps/earth/vectorosm").arg(MarbleDirs::localPath())},
0151                           {{"e", "extension"}, "Output file type: o5m (default), osm or kml", "file extension", "o5m"}
0152                       });
0153
0154     // Process the actual command line arguments given by the user
0155     parser.process(app);
0156
0157     const QStringList args = parser.positionalArguments();
0158     if (args.isEmpty()) {
0159         parser.showHelp();
0160         return 0;
0161     }
0162     // input is args.at(0), output is args.at(1)
0163
0164     QString const extension = parser.value("extension");
0165     QString inputFileName = args.at(0);
0166     auto const levels = parser.value("zoom-level").split(',');
0167     QVector<unsigned int> zoomLevels;
0168     int maxZoomLevel = 0;
0169     for(auto const &level: levels) {
0170         int const zoomLevel = level.toInt();
0171         maxZoomLevel = qMax(zoomLevel, maxZoomLevel);
0172         zoomLevels << zoomLevel;
0173     }
0174
0175     if (zoomLevels.isEmpty()) {
0176         parser.showHelp(1);
0177         return 1;
0178     }
0179
0180     // work around MARBLE_ADD_WRITER not working for static builds
0181 #ifdef STATIC_BUILD
0182     GeoDataDocumentWriter::registerWriter(new O5mWriter, QStringLiteral("o5m"));
0183 #endif
0184
0185     bool const overwriteTiles = parser.value("conflict-resolution") == "overwrite";
0186     bool const mergeTiles = parser.value("conflict-resolution") == "merge";
0187     bool const writeBoundaries = mergeTiles || parser.isSet("boundaries");
0188     QSharedPointer<MbTileWriter> mbtileWriter;
0189     if (parser.isSet("mbtile")) {
0190         QString const mbtile = parser.value("mbtile");
0191         mbtileWriter = QSharedPointer<MbTileWriter>(new MbTileWriter(mbtile, extension));
0192         mbtileWriter->setReportProgress(false);
0193         mbtileWriter->setCommitInterval(500);
0194     }
0195
0196     MarbleModel model;
0197     ParsingRunnerManager manager(model.pluginManager());
0198     QString const cacheDirectory = parser.value("cache-directory");
0199     QDir().mkpath(cacheDirectory);
0200     if (!QFileInfo(cacheDirectory).isWritable()) {
0201         qWarning() << "Cannot write to cache directory" << cacheDirectory;
0202         parser.showHelp(1);
0203     }
0204
0205     if (*zoomLevels.cbegin() <= 9) {
0206         auto map = TileDirectory::open(inputFileName, manager);
0207         VectorClipper processor(map.data(), maxZoomLevel);
0208         GeoDataLatLonBox world(85.0, -85.0, 180.0, -180.0, GeoDataCoordinates::Degree);
0209         if (parser.isSet("spellcheck")) {
0210             SpellChecker spellChecker(parser.value("spellcheck"));
0211             spellChecker.setVerbose(parser.isSet("verbose"));
0212             spellChecker.correctPlaceLabels(map.data()->placemarkList());
0213         }
0214         for(auto zoomLevel: zoomLevels) {
0215             TileIterator iter(world, zoomLevel);
0216             qint64 count = 0;
0217             qint64 const total = iter.total();
0218             for(auto const &tileId: iter) {
0219                 ++count;
0220                 QString const filename = tileFileName(parser, tileId.x(), tileId.y(), zoomLevel);
0221                 if (!overwriteTiles && QFileInfo(filename).exists()) {
0222                     continue;
0223                 }
0224                 GeoDataDocument* tile = processor.clipTo(zoomLevel, tileId.x(), tileId.y());
0225                 if (!tile->isEmpty()) {
0226                     NodeReducer nodeReducer(tile, TileId(0, zoomLevel, tileId.x(), tileId.y()));
0227                     if (!writeTile(tile, filename)) {
0228                         return 4;
0229                     }
0230                     TileDirectory::printProgress(count / double(total));
0231                     std::cout << " Tile " << count << "/" << total << " (" << tile->name().toStdString() << ") done.";
0232                     double const reduction = nodeReducer.removedNodes() / qMax(1.0, double(nodeReducer.remainingNodes() + nodeReducer.removedNodes()));
0233                     std::cout << " Node reduction: " << qRound(reduction * 100.0) << "%";
0234                 } else {
0235                     TileDirectory::printProgress(count / double(total));
0236                     std::cout << " Skipping empty tile " << count << "/" << total << " (" << tile->name().toStdString() << ").";
0237                 }
0238                 std::cout << std::string(20, ' ') << '\r';
0239                 std::cout.flush();
0240                 delete tile;
0241             }
0242         }
0243     } else {
0244         QString const region = QFileInfo(inputFileName).fileName();
0245         QString const regionDir = QString("%1/%2").arg(cacheDirectory).arg(QFileInfo(inputFileName).baseName());
0246         TileDirectory mapTiles(TileDirectory::OpenStreetMap, regionDir, manager, maxZoomLevel);
0247         mapTiles.setInputFile(inputFileName);
0248         mapTiles.createTiles();
0249         auto const boundingBox = mapTiles.boundingBox();
0250
0251         TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, maxZoomLevel);
0252         loader.setBoundingBox(boundingBox);
0253         loader.createTiles();
0254
0255         typedef QMap<QString, QVector<TileId> > Tiles;
0256         Tiles tiles;
0257
0258         qint64 total = 0;
0259         QSet<QString> boundaryTiles;
0260         for(auto zoomLevel: zoomLevels) {
0261             TileIterator iter(mapTiles.boundingBox(), zoomLevel);
0262             total += iter.total();
0263             for(auto const &tileId: iter) {
0264                 auto const tile = TileId(0, zoomLevel, tileId.x(), tileId.y());
0265                 int const innerNodes = mapTiles.innerNodes(tile);
0266                 if (innerNodes > 0) {
0267                     auto const mapTile = mapTiles.tileFor(zoomLevel, tileId.x(), tileId.y());
0268                     auto const name = QString("%1/%2/%3").arg(mapTile.zoomLevel()).arg(mapTile.x()).arg(mapTile.y());
0269                     tiles[name] << tile;
0270                     if (innerNodes < 4) {
0271                         boundaryTiles << name;
0272                     }
0273                 } else {
0274                     --total;
0275                 }
0276             }
0277         }
0278
0279         qint64 count = 0;
0280         for (auto iter = tiles.begin(), end = tiles.end(); iter != end; ++iter) {
0281             for(auto const &tileId: iter.value()) {
0282                 ++count;
0283                 int const zoomLevel = tileId.zoomLevel();
0284                 QString const filename = tileFileName(parser, tileId.x(), tileId.y(), zoomLevel);
0285                 if (!overwriteTiles) {
0286                     if (zoomLevel > 13 && mbtileWriter && mbtileWriter->hasTile(tileId.x(), tileId.y(), zoomLevel)) {
0287                         continue;
0288                     } else if (QFileInfo(filename).exists()) {
0289                         continue;
0290                     }
0291                 }
0292
0293                 using GeoDocPtr = QSharedPointer<GeoDataDocument>;
0294                 GeoDocPtr tile2 = GeoDocPtr(loader.clip(zoomLevel, tileId.x(), tileId.y()));
0295                 if (!tile2->isEmpty()) {
0296                     GeoDocPtr tile1 = GeoDocPtr(mapTiles.clip(zoomLevel, tileId.x(), tileId.y()));
0297                     TagsFilter::removeAnnotationTags(tile1.data());
0298                     int originalWays = 0;
0299                     int mergedWays = 0;
0300                     if (zoomLevel < 17) {
0301                         WayConcatenator concatenator(tile1.data());
0302                         originalWays = concatenator.originalWays();
0303                         mergedWays = concatenator.mergedWays();
0304                     }
0305                     NodeReducer nodeReducer(tile1.data(), tileId);
0306                     if (!tile1->isEmpty() && !tile2->isEmpty()) {
0307                         GeoDocPtr combined = GeoDocPtr(mergeDocuments(tile1.data(), tile2.data()));
0308
0309                         if (writeBoundaries && boundaryTiles.contains(iter.key())) {
0310                             writeBoundaryTile(tile1.data(), region, parser, tileId.x(), tileId.y(), zoomLevel);
0311                             if (mergeTiles) {
0312                                 combined = mergeBoundaryTiles(tile2, manager, parser, tileId.x(), tileId.y(), zoomLevel);
0313                             }
0314                         }
0315
0316                         if (zoomLevel > 13 && mbtileWriter) {
0317                             QBuffer buffer;
0318                             buffer.open(QBuffer::ReadWrite);
0319                             if (GeoDataDocumentWriter::write(&buffer, *combined, extension)) {
0320                                 buffer.seek(0);
0321                                 mbtileWriter->addTile(&buffer, tileId.x(), tileId.y(), zoomLevel);
0322                             } else {
0323                                 qWarning() << "Could not write the tile " << combined->name();
0324                             }
0325                         } else {
0326                             if (!writeTile(combined.data(), filename)) {
0327                                 return 4;
0328                             }
0329                         }
0330
0331                         TileDirectory::printProgress(count / double(total));
0332                         std::cout << "  Tile " << count << "/" << total << " (";
0333                         std::cout << combined->name().toStdString() << ").";
0334                         double const reduction = nodeReducer.removedNodes() / qMax(1.0, double(nodeReducer.remainingNodes() + nodeReducer.removedNodes()));
0335                         std::cout << " Node reduction: " << qRound(reduction * 100.0) << "%";
0336                         if (originalWays > 0) {
0337                             std::cout << " , " << originalWays << " ways merged to " << mergedWays;
0338                         }
0339                     } else {
0340                         TileDirectory::printProgress(count / double(total));
0341                         std::cout << "  Skipping empty tile " << count << "/" << total << " (" << tile1->name().toStdString() << ").";
0342                     }
0343                 } else {
0344                     TileDirectory::printProgress(count / double(total));
0345                     std::cout << "  Skipping sea tile " << count << "/" << total << " (" << tile2->name().toStdString() << ").";
0346                 }
0347
0348                 std::cout << std::string(20, ' ') << '\r';
0349                 std::cout.flush();
0350             }
0351         }
0352         TileDirectory::printProgress(1.0);
0353         std::cout << "  Vector OSM tiles complete." << std::string(30, ' ') << std::endl;
0354     }
0355
0356     return 0;
0357 }