File indexing completed on 2024-10-27 09:52:30

0001 // SPDX-FileCopyrightText: 2011 Dennis Nienhüser <nienhueser@kde.org>
0002 //
0003 // SPDX-License-Identifier: GPL-3.0-or-later
0004 //
0005 // This file originates from the MoNav project where it was named pbfreader.h and
0006 // SPDX-FileCopyrightText: 2010 Christian Vetter <veaac.fdirct@gmail.com>
0007 //
0008 
0009 #include "PbfParser.h"
0010 
0011 #include <QDebug>
0012 
0013 #include <zlib.h>
0014 
0015 #include <fstream>
0016 
0017 #include <netinet/in.h>
0018 
0019 using namespace std;
0020 using namespace OSMPBF;
0021 
0022 PbfParser::PbfParser() :
0023     m_currentGroup( 0 ),
0024     m_currentEntity( 0 ),
0025     m_loadBlock( false )
0026 {
0027     GOOGLE_PROTOBUF_VERIFY_VERSION;
0028 }
0029 
0030 bool PbfParser::parse( const QFileInfo &fileInfo, int pass, bool &needAnotherPass )
0031 {
0032     m_pass = pass;
0033     needAnotherPass = pass < 2;
0034 
0035     QFile file( fileInfo.absoluteFilePath() );
0036     if ( !file.open( QFile::ReadOnly ) ) {
0037         qCritical() << "Unable to open file " << fileInfo.absoluteFilePath() << " for reading.";
0038         return false;
0039     }
0040 
0041     m_stream.setDevice( &file );
0042     m_stream.setByteOrder( QDataStream::BigEndian );
0043 
0044     if ( !parseBlobHeader() ) {
0045         return false;
0046     }
0047 
0048     if ( m_blobHeader.type() != "OSMHeader" ) {
0049         qCritical() << "Unable to parse blob header type " << m_blobHeader.type().c_str();
0050         return false;
0051     }
0052 
0053     if ( !parseBlob() ) {
0054         return false;
0055     }
0056 
0057     if ( !parseData() ) {
0058         return false;
0059     }
0060 
0061     m_loadBlock = true;
0062 
0063     while ( true ) {
0064 
0065         if ( m_loadBlock ) {
0066             if ( !readNext() ) {
0067                 if ( pass == 1 ) {
0068                     m_referencedWays.clear();
0069                 } else if ( pass == 2 ) {
0070                     m_referencedNodes.clear();
0071                 }
0072 
0073                 return true;
0074             }
0075             loadBlock();
0076             loadGroup();
0077         }
0078 
0079         switch ( m_mode ) {
0080         case ModeNode:
0081             parseNode();
0082             break;
0083         case ModeWay:
0084             parseWay();
0085             break;
0086         case ModeRelation:
0087             parseRelation();
0088             break;
0089         case ModeDense:
0090             parseDense();
0091             break;
0092         }
0093     }
0094 
0095     return true;
0096 }
0097 
0098 bool PbfParser::parseBlobHeader()
0099 {
0100     int size( -1 );
0101     m_stream >> size;
0102 
0103     if ( size < 0 ) {
0104         qCritical() << "Invalid blob header size " << size;
0105         return false;
0106     }
0107 
0108     m_buffer.resize( size );
0109     int readBytes = m_stream.readRawData( m_buffer.data(), size );
0110     if ( readBytes != size ) {
0111         qCritical() << "Unable to read blob header";
0112         return false;
0113     }
0114 
0115     if ( !m_blobHeader.ParseFromArray( m_buffer.constData(), size ) ) {
0116         qCritical() << "Unable to parse blob header";
0117         return false;
0118     }
0119 
0120     return true;
0121 }
0122 
0123 bool PbfParser::parseBlob()
0124 {
0125     int size = m_blobHeader.datasize();
0126     if ( size < 0 ) {
0127         qCritical() << "invalid blob size:" << size;
0128         return false;
0129     }
0130 
0131     m_buffer.resize( size );
0132     int readBytes = m_stream.readRawData( m_buffer.data(), size );
0133     if ( readBytes != size ) {
0134         qCritical() << "failed to read blob";
0135         return false;
0136     }
0137 
0138     if ( !m_blob.ParseFromArray( m_buffer.constData(), size ) ) {
0139         qCritical() << "failed to parse blob";
0140         return false;
0141     }
0142 
0143     if ( m_blob.has_raw() ) {
0144         const std::string& data = m_blob.raw();
0145         m_buffer.resize( data.size() );
0146         for ( unsigned int i = 0; i < data.size(); ++i ) {
0147             m_buffer[i] = data[i];
0148         }
0149     } else if ( m_blob.has_zlib_data() ) {
0150         m_buffer.resize( m_blob.raw_size() );
0151         z_stream zStream;
0152         zStream.next_in = ( unsigned char* ) m_blob.zlib_data().data();
0153         zStream.avail_in = m_blob.zlib_data().size();
0154         zStream.next_out = ( unsigned char* ) m_buffer.data();
0155         zStream.avail_out = m_blob.raw_size();
0156         zStream.zalloc = Z_NULL;
0157         zStream.zfree = Z_NULL;
0158         zStream.opaque = Z_NULL;
0159         int result = inflateInit( &zStream );
0160         if ( result != Z_OK ) {
0161             qCritical() << "failed to open zlib m_stream";
0162             return false;
0163         }
0164         result = inflate( &zStream, Z_FINISH );
0165         if ( result != Z_STREAM_END ) {
0166             qCritical() << "failed to inflate zlib m_stream";
0167             return false;
0168         }
0169         result = inflateEnd( &zStream );
0170         if ( result != Z_OK ) {
0171             qCritical() << "failed to close zlib m_stream";
0172             return false;
0173         }
0174 
0175         return true;
0176     } else if ( m_blob.has_lzma_data() ) {
0177         qCritical() << "No support for lzma decryption implemented, sorry.";
0178         return false;
0179     } else {
0180         qCritical() << "Blob contains no data";
0181         return false;
0182     }
0183 
0184     return true;
0185 }
0186 
0187 bool PbfParser::parseData()
0188 {
0189     if ( !m_headerBlock.ParseFromArray( m_buffer.data(), m_buffer.size() ) ) {
0190         qCritical() << "failed to parse header block";
0191         return false;
0192     }
0193 
0194     for ( int i = 0; i < m_headerBlock.required_features_size(); ++i ) {
0195         string const & feature = m_headerBlock.required_features( i );
0196         if ( feature != "OsmSchema-V0.6" && feature != "DenseNodes" ) {
0197             qCritical() << "Support for feature " << feature.c_str() << "not implemented";
0198             return false;
0199         }
0200     }
0201 
0202     return true;
0203 }
0204 
0205 bool PbfParser::readNext()
0206 {
0207     if ( !parseBlobHeader() )
0208         return false;
0209 
0210     if ( m_blobHeader.type() != "OSMData" ) {
0211         qCritical() << "invalid block type, found" << m_blobHeader.type().data() << "instead of OSMData";
0212         return false;
0213     }
0214 
0215     if ( !parseBlob() )
0216         return false;
0217 
0218     if ( !m_primitiveBlock.ParseFromArray( m_buffer.data(), m_buffer.size() ) ) {
0219         qCritical() << "failed to parse PrimitiveBlock";
0220         return false;
0221     }
0222     return true;
0223 }
0224 
0225 void PbfParser::loadGroup()
0226 {
0227     const PrimitiveGroup& group = m_primitiveBlock.primitivegroup( m_currentGroup );
0228     if ( group.nodes_size() != 0 ) {
0229         m_mode = ModeNode;
0230     } else if ( group.ways_size() != 0 ) {
0231         m_mode = ModeWay;
0232     } else if ( group.relations_size() != 0 ) {
0233         m_mode = ModeRelation;
0234     } else if ( group.has_dense() )  {
0235         m_mode = ModeDense;
0236         m_lastDenseID = 0;
0237         m_lastDenseTag = 0;
0238         m_lastDenseLatitude = 0;
0239         m_lastDenseLongitude = 0;
0240         assert( group.dense().id_size() != 0 );
0241     } else
0242         assert( false );
0243 }
0244 
0245 void PbfParser::loadBlock()
0246 {
0247     m_loadBlock = false;
0248     m_currentGroup = 0;
0249     m_currentEntity = 0;
0250 }
0251 
0252 void PbfParser::parseNode()
0253 {
0254     if ( m_pass == 2 ) {
0255         const Node& inputNode = m_primitiveBlock.primitivegroup( m_currentGroup ).nodes( m_currentEntity );
0256         Marble::Node node;
0257         node.lat = ( ( double ) inputNode.lat() * m_primitiveBlock.granularity() + m_primitiveBlock.lat_offset() ) / ( 1000.0 * 1000.0 * 1000.0 );
0258         node.lon = ( ( double ) inputNode.lon() * m_primitiveBlock.granularity() + m_primitiveBlock.lon_offset() ) / ( 1000.0 * 1000.0 * 1000.0 );
0259 
0260         for ( int tag = 0; tag < inputNode.keys_size(); tag++ ) {
0261 
0262             QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputNode.keys( tag ) ).data() );
0263             QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputNode.vals( tag ) ).data() );
0264 
0265             if (key == QLatin1String("name")) {
0266                 node.name = value.trimmed();
0267             } else if (key == QLatin1String("addr:street")) {
0268                 node.street = value.trimmed();
0269                 node.save = true;
0270             } else if (key == QLatin1String("addr:housenumber")) {
0271                 node.houseNumber = value;
0272                 node.save = true;
0273             } else if (key == QLatin1String("addr:city")) {
0274                 node.city = value;
0275                 node.save = true;
0276             } else {
0277                 if ( shouldSave( Marble::NodeType, key, value ) ) {
0278                     node.save = true;
0279                 }
0280                 setCategory( node, key, value );
0281             }
0282         }
0283 
0284         if ( node.save ) {
0285             m_nodes[inputNode.id()] = node;
0286         }
0287 
0288         if ( m_referencedNodes.contains( inputNode.id() ) ) {
0289             m_coordinates[inputNode.id()] = node;
0290         }
0291     }
0292 
0293     m_currentEntity++;
0294     if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).nodes_size() ) {
0295         m_currentEntity = 0;
0296         m_currentGroup++;
0297         if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() )
0298             m_loadBlock = true;
0299         else
0300             loadGroup();
0301     }
0302 }
0303 
0304 void PbfParser::parseWay()
0305 {
0306     if ( m_pass == 1 ) {
0307         const Way& inputWay = m_primitiveBlock.primitivegroup( m_currentGroup ).ways( m_currentEntity );
0308         Marble::Way way;
0309         Marble::Relation relation;
0310 
0311         for ( int tag = 0; tag < inputWay.keys_size(); tag++ ) {
0312             QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputWay.keys( tag ) ).data() );
0313             QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputWay.vals( tag ) ).data() );
0314 
0315             if (key == QLatin1String("name")) {
0316                 way.name = value.trimmed();
0317             } else if (key == QLatin1String("addr:street")) {
0318                 way.street = value.trimmed();
0319                 way.save = true;
0320             } else if (key == QLatin1String("addr:housenumber")) {
0321                 way.houseNumber = value;
0322                 way.save = true;
0323             } else if (key == QLatin1String("addr:city")) {
0324                 way.city = value;
0325                 way.save = true;
0326             } else if (key == QLatin1String("building") && value == QLatin1String("yes")) {
0327                 way.isBuilding = true;
0328             } else if (key == QLatin1String("boundary") && value == QLatin1String("administrative")) {
0329                 relation.isAdministrativeBoundary = true;
0330             } else if (key == QLatin1String("admin_level")) {
0331                 relation.adminLevel = value.toInt();
0332             } else  {
0333                 if ( shouldSave( Marble::WayType, key, value ) ) {
0334                     way.save = true;
0335                 }
0336                 setCategory( way, key, value );
0337             }
0338         }
0339 
0340         long long lastRef = 0;
0341         for ( int i = 0; i < inputWay.refs_size(); i++ ) {
0342             lastRef += inputWay.refs( i );
0343             way.nodes.push_back( lastRef );
0344         }
0345 
0346         if ( relation.isAdministrativeBoundary && !way.name.isEmpty() ) {
0347             relation.name = way.name;
0348             relation.ways << QPair<int, Marble::RelationRole>( inputWay.id(), Marble::Outer );
0349             m_relations[inputWay.id()] = relation;
0350         }
0351 
0352         if ( way.save || m_referencedWays.contains( inputWay.id() ) ) {
0353             if ( !way.isBuilding && way.nodes.size() > 1 && !m_referencedWays.contains( inputWay.id() ) ) {
0354                 QList<int> nodes = way.nodes;
0355                 way.nodes.clear();
0356                 way.nodes << nodes.first();
0357                 if ( nodes.size() > 2 ) {
0358                     way.nodes << nodes.at( nodes.size() / 2 );
0359                 }
0360                 way.nodes << nodes.last();
0361             }
0362 
0363             for( int node: way.nodes ) {
0364                 m_referencedNodes << node;
0365             }
0366 
0367             m_ways[inputWay.id()] = way;
0368         }
0369     }
0370 
0371     m_currentEntity++;
0372     if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).ways_size() ) {
0373         m_currentEntity = 0;
0374         m_currentGroup++;
0375         if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() )
0376             m_loadBlock = true;
0377         else
0378             loadGroup();
0379     }
0380 }
0381 
0382 void PbfParser::parseRelation()
0383 {
0384     if ( m_pass == 0 ) {
0385         const Relation& inputRelation = m_primitiveBlock.primitivegroup( m_currentGroup ).relations( m_currentEntity );
0386         Marble::Relation relation;
0387 
0388         for ( int tag = 0; tag < inputRelation.keys_size(); tag++ ) {
0389 
0390             QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputRelation.keys( tag ) ).data() );
0391             QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputRelation.vals( tag ) ).data() );
0392 
0393             if (key == QLatin1String("boundary") && value == QLatin1String("administrative")) {
0394                 relation.isAdministrativeBoundary = true;
0395             } else if (key == QLatin1String("admin_level")) {
0396                 relation.adminLevel = value.toInt();
0397             } else if (key == QLatin1String("name")) {
0398                 relation.name = value.trimmed();
0399             } else if (key == QLatin1String("type") && value == QLatin1String("multipolygon")) {
0400                 relation.isMultipolygon = true;
0401             }
0402         }
0403 
0404         if ( relation.isAdministrativeBoundary ) {
0405             long long lastRef = 0;
0406             for ( int i = 0; i < inputRelation.types_size(); i++ ) {
0407                 lastRef += inputRelation.memids( i );
0408                 switch ( inputRelation.types( i ) ) {
0409                 case OSMPBF::Relation::NODE:
0410                     relation.nodes.push_back( lastRef );
0411                     break;
0412                 case OSMPBF::Relation::WAY: {
0413                     string role = m_primitiveBlock.stringtable().s( inputRelation.roles_sid( i ) ).data();
0414                     Marble::RelationRole relationRole = Marble::None;
0415                     if ( role == "outer" ) relationRole = Marble::Outer;
0416                     if ( role == "inner" ) relationRole = Marble::Inner;
0417                     m_referencedWays << lastRef;
0418                     relation.ways.push_back( QPair<int, Marble::RelationRole>( lastRef, relationRole ) );
0419                 }
0420                 break;
0421                 case OSMPBF::Relation::RELATION:
0422                     relation.relations.push_back( lastRef );
0423                 }
0424             }
0425 
0426             m_relations[inputRelation.id()] = relation;
0427         }
0428     }
0429 
0430     m_currentEntity++;
0431     if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).relations_size() ) {
0432         m_currentEntity = 0;
0433         m_currentGroup++;
0434         if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() )
0435             m_loadBlock = true;
0436         else
0437             loadGroup();
0438     }
0439 }
0440 
0441 void PbfParser::parseDense()
0442 {
0443     const DenseNodes& dense = m_primitiveBlock.primitivegroup( m_currentGroup ).dense();
0444     if ( m_pass == 2 ) {
0445         m_lastDenseID += dense.id( m_currentEntity );
0446         m_lastDenseLatitude += dense.lat( m_currentEntity );
0447         m_lastDenseLongitude += dense.lon( m_currentEntity );
0448 
0449         Marble::Node node;
0450         node.lat = ( ( double ) m_lastDenseLatitude * m_primitiveBlock.granularity() + m_primitiveBlock.lat_offset() ) / ( 1000.0 * 1000.0 * 1000.0 );
0451         node.lon = ( ( double ) m_lastDenseLongitude * m_primitiveBlock.granularity() + m_primitiveBlock.lon_offset() ) / ( 1000.0 * 1000.0 * 1000.0 );
0452 
0453         while ( true ) {
0454             if ( m_lastDenseTag >= dense.keys_vals_size() )
0455                 break;
0456 
0457             int tagValue = dense.keys_vals( m_lastDenseTag );
0458             if ( tagValue == 0 ) {
0459                 m_lastDenseTag++;
0460                 break;
0461             }
0462 
0463             QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( dense.keys_vals( m_lastDenseTag ) ).data() );
0464             QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( dense.keys_vals( m_lastDenseTag + 1 ) ).data() );
0465 
0466             if (key == QLatin1String("name")) {
0467                 node.name = value.trimmed();
0468             } else if (key == QLatin1String("addr:street")) {
0469                 node.street = value.trimmed();
0470                 node.save = true;
0471             } else if (key == QLatin1String("addr:housenumber")) {
0472                 node.houseNumber = value;
0473                 node.save = true;
0474             } else if (key == QLatin1String("addr:city")) {
0475                 node.city = value;
0476                 node.save = true;
0477             } else {
0478                 if ( shouldSave( Marble::NodeType, key, value ) ) {
0479                     node.save = true;
0480                 }
0481                 setCategory( node, key, value );
0482             }
0483 
0484             m_lastDenseTag += 2;
0485         }
0486 
0487         if ( node.save) {
0488             m_nodes[m_lastDenseID] = node;
0489         }
0490 
0491         if ( m_referencedNodes.contains( m_lastDenseID ) ) {
0492             m_coordinates[m_lastDenseID] = node;
0493         }
0494     }
0495 
0496     ++m_currentEntity;
0497     if ( m_currentEntity >= dense.id_size() ) {
0498         m_currentEntity = 0;
0499         m_currentGroup++;
0500         if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() ) {
0501             m_loadBlock = true;
0502         } else {
0503             loadGroup();
0504         }
0505     }
0506 }