File indexing completed on 2024-10-27 09:52:30
0001 // SPDX-FileCopyrightText: 2011 Dennis Nienhüser <nienhueser@kde.org> 0002 // 0003 // SPDX-License-Identifier: GPL-3.0-or-later 0004 // 0005 // This file originates from the MoNav project where it was named pbfreader.h and 0006 // SPDX-FileCopyrightText: 2010 Christian Vetter <veaac.fdirct@gmail.com> 0007 // 0008 0009 #include "PbfParser.h" 0010 0011 #include <QDebug> 0012 0013 #include <zlib.h> 0014 0015 #include <fstream> 0016 0017 #include <netinet/in.h> 0018 0019 using namespace std; 0020 using namespace OSMPBF; 0021 0022 PbfParser::PbfParser() : 0023 m_currentGroup( 0 ), 0024 m_currentEntity( 0 ), 0025 m_loadBlock( false ) 0026 { 0027 GOOGLE_PROTOBUF_VERIFY_VERSION; 0028 } 0029 0030 bool PbfParser::parse( const QFileInfo &fileInfo, int pass, bool &needAnotherPass ) 0031 { 0032 m_pass = pass; 0033 needAnotherPass = pass < 2; 0034 0035 QFile file( fileInfo.absoluteFilePath() ); 0036 if ( !file.open( QFile::ReadOnly ) ) { 0037 qCritical() << "Unable to open file " << fileInfo.absoluteFilePath() << " for reading."; 0038 return false; 0039 } 0040 0041 m_stream.setDevice( &file ); 0042 m_stream.setByteOrder( QDataStream::BigEndian ); 0043 0044 if ( !parseBlobHeader() ) { 0045 return false; 0046 } 0047 0048 if ( m_blobHeader.type() != "OSMHeader" ) { 0049 qCritical() << "Unable to parse blob header type " << m_blobHeader.type().c_str(); 0050 return false; 0051 } 0052 0053 if ( !parseBlob() ) { 0054 return false; 0055 } 0056 0057 if ( !parseData() ) { 0058 return false; 0059 } 0060 0061 m_loadBlock = true; 0062 0063 while ( true ) { 0064 0065 if ( m_loadBlock ) { 0066 if ( !readNext() ) { 0067 if ( pass == 1 ) { 0068 m_referencedWays.clear(); 0069 } else if ( pass == 2 ) { 0070 m_referencedNodes.clear(); 0071 } 0072 0073 return true; 0074 } 0075 loadBlock(); 0076 loadGroup(); 0077 } 0078 0079 switch ( m_mode ) { 0080 case ModeNode: 0081 parseNode(); 0082 break; 0083 case ModeWay: 0084 parseWay(); 0085 break; 0086 case ModeRelation: 0087 parseRelation(); 0088 break; 0089 case ModeDense: 0090 parseDense(); 0091 break; 0092 } 0093 } 0094 0095 return true; 0096 } 0097 0098 bool PbfParser::parseBlobHeader() 0099 { 0100 int size( -1 ); 0101 m_stream >> size; 0102 0103 if ( size < 0 ) { 0104 qCritical() << "Invalid blob header size " << size; 0105 return false; 0106 } 0107 0108 m_buffer.resize( size ); 0109 int readBytes = m_stream.readRawData( m_buffer.data(), size ); 0110 if ( readBytes != size ) { 0111 qCritical() << "Unable to read blob header"; 0112 return false; 0113 } 0114 0115 if ( !m_blobHeader.ParseFromArray( m_buffer.constData(), size ) ) { 0116 qCritical() << "Unable to parse blob header"; 0117 return false; 0118 } 0119 0120 return true; 0121 } 0122 0123 bool PbfParser::parseBlob() 0124 { 0125 int size = m_blobHeader.datasize(); 0126 if ( size < 0 ) { 0127 qCritical() << "invalid blob size:" << size; 0128 return false; 0129 } 0130 0131 m_buffer.resize( size ); 0132 int readBytes = m_stream.readRawData( m_buffer.data(), size ); 0133 if ( readBytes != size ) { 0134 qCritical() << "failed to read blob"; 0135 return false; 0136 } 0137 0138 if ( !m_blob.ParseFromArray( m_buffer.constData(), size ) ) { 0139 qCritical() << "failed to parse blob"; 0140 return false; 0141 } 0142 0143 if ( m_blob.has_raw() ) { 0144 const std::string& data = m_blob.raw(); 0145 m_buffer.resize( data.size() ); 0146 for ( unsigned int i = 0; i < data.size(); ++i ) { 0147 m_buffer[i] = data[i]; 0148 } 0149 } else if ( m_blob.has_zlib_data() ) { 0150 m_buffer.resize( m_blob.raw_size() ); 0151 z_stream zStream; 0152 zStream.next_in = ( unsigned char* ) m_blob.zlib_data().data(); 0153 zStream.avail_in = m_blob.zlib_data().size(); 0154 zStream.next_out = ( unsigned char* ) m_buffer.data(); 0155 zStream.avail_out = m_blob.raw_size(); 0156 zStream.zalloc = Z_NULL; 0157 zStream.zfree = Z_NULL; 0158 zStream.opaque = Z_NULL; 0159 int result = inflateInit( &zStream ); 0160 if ( result != Z_OK ) { 0161 qCritical() << "failed to open zlib m_stream"; 0162 return false; 0163 } 0164 result = inflate( &zStream, Z_FINISH ); 0165 if ( result != Z_STREAM_END ) { 0166 qCritical() << "failed to inflate zlib m_stream"; 0167 return false; 0168 } 0169 result = inflateEnd( &zStream ); 0170 if ( result != Z_OK ) { 0171 qCritical() << "failed to close zlib m_stream"; 0172 return false; 0173 } 0174 0175 return true; 0176 } else if ( m_blob.has_lzma_data() ) { 0177 qCritical() << "No support for lzma decryption implemented, sorry."; 0178 return false; 0179 } else { 0180 qCritical() << "Blob contains no data"; 0181 return false; 0182 } 0183 0184 return true; 0185 } 0186 0187 bool PbfParser::parseData() 0188 { 0189 if ( !m_headerBlock.ParseFromArray( m_buffer.data(), m_buffer.size() ) ) { 0190 qCritical() << "failed to parse header block"; 0191 return false; 0192 } 0193 0194 for ( int i = 0; i < m_headerBlock.required_features_size(); ++i ) { 0195 string const & feature = m_headerBlock.required_features( i ); 0196 if ( feature != "OsmSchema-V0.6" && feature != "DenseNodes" ) { 0197 qCritical() << "Support for feature " << feature.c_str() << "not implemented"; 0198 return false; 0199 } 0200 } 0201 0202 return true; 0203 } 0204 0205 bool PbfParser::readNext() 0206 { 0207 if ( !parseBlobHeader() ) 0208 return false; 0209 0210 if ( m_blobHeader.type() != "OSMData" ) { 0211 qCritical() << "invalid block type, found" << m_blobHeader.type().data() << "instead of OSMData"; 0212 return false; 0213 } 0214 0215 if ( !parseBlob() ) 0216 return false; 0217 0218 if ( !m_primitiveBlock.ParseFromArray( m_buffer.data(), m_buffer.size() ) ) { 0219 qCritical() << "failed to parse PrimitiveBlock"; 0220 return false; 0221 } 0222 return true; 0223 } 0224 0225 void PbfParser::loadGroup() 0226 { 0227 const PrimitiveGroup& group = m_primitiveBlock.primitivegroup( m_currentGroup ); 0228 if ( group.nodes_size() != 0 ) { 0229 m_mode = ModeNode; 0230 } else if ( group.ways_size() != 0 ) { 0231 m_mode = ModeWay; 0232 } else if ( group.relations_size() != 0 ) { 0233 m_mode = ModeRelation; 0234 } else if ( group.has_dense() ) { 0235 m_mode = ModeDense; 0236 m_lastDenseID = 0; 0237 m_lastDenseTag = 0; 0238 m_lastDenseLatitude = 0; 0239 m_lastDenseLongitude = 0; 0240 assert( group.dense().id_size() != 0 ); 0241 } else 0242 assert( false ); 0243 } 0244 0245 void PbfParser::loadBlock() 0246 { 0247 m_loadBlock = false; 0248 m_currentGroup = 0; 0249 m_currentEntity = 0; 0250 } 0251 0252 void PbfParser::parseNode() 0253 { 0254 if ( m_pass == 2 ) { 0255 const Node& inputNode = m_primitiveBlock.primitivegroup( m_currentGroup ).nodes( m_currentEntity ); 0256 Marble::Node node; 0257 node.lat = ( ( double ) inputNode.lat() * m_primitiveBlock.granularity() + m_primitiveBlock.lat_offset() ) / ( 1000.0 * 1000.0 * 1000.0 ); 0258 node.lon = ( ( double ) inputNode.lon() * m_primitiveBlock.granularity() + m_primitiveBlock.lon_offset() ) / ( 1000.0 * 1000.0 * 1000.0 ); 0259 0260 for ( int tag = 0; tag < inputNode.keys_size(); tag++ ) { 0261 0262 QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputNode.keys( tag ) ).data() ); 0263 QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputNode.vals( tag ) ).data() ); 0264 0265 if (key == QLatin1String("name")) { 0266 node.name = value.trimmed(); 0267 } else if (key == QLatin1String("addr:street")) { 0268 node.street = value.trimmed(); 0269 node.save = true; 0270 } else if (key == QLatin1String("addr:housenumber")) { 0271 node.houseNumber = value; 0272 node.save = true; 0273 } else if (key == QLatin1String("addr:city")) { 0274 node.city = value; 0275 node.save = true; 0276 } else { 0277 if ( shouldSave( Marble::NodeType, key, value ) ) { 0278 node.save = true; 0279 } 0280 setCategory( node, key, value ); 0281 } 0282 } 0283 0284 if ( node.save ) { 0285 m_nodes[inputNode.id()] = node; 0286 } 0287 0288 if ( m_referencedNodes.contains( inputNode.id() ) ) { 0289 m_coordinates[inputNode.id()] = node; 0290 } 0291 } 0292 0293 m_currentEntity++; 0294 if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).nodes_size() ) { 0295 m_currentEntity = 0; 0296 m_currentGroup++; 0297 if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() ) 0298 m_loadBlock = true; 0299 else 0300 loadGroup(); 0301 } 0302 } 0303 0304 void PbfParser::parseWay() 0305 { 0306 if ( m_pass == 1 ) { 0307 const Way& inputWay = m_primitiveBlock.primitivegroup( m_currentGroup ).ways( m_currentEntity ); 0308 Marble::Way way; 0309 Marble::Relation relation; 0310 0311 for ( int tag = 0; tag < inputWay.keys_size(); tag++ ) { 0312 QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputWay.keys( tag ) ).data() ); 0313 QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputWay.vals( tag ) ).data() ); 0314 0315 if (key == QLatin1String("name")) { 0316 way.name = value.trimmed(); 0317 } else if (key == QLatin1String("addr:street")) { 0318 way.street = value.trimmed(); 0319 way.save = true; 0320 } else if (key == QLatin1String("addr:housenumber")) { 0321 way.houseNumber = value; 0322 way.save = true; 0323 } else if (key == QLatin1String("addr:city")) { 0324 way.city = value; 0325 way.save = true; 0326 } else if (key == QLatin1String("building") && value == QLatin1String("yes")) { 0327 way.isBuilding = true; 0328 } else if (key == QLatin1String("boundary") && value == QLatin1String("administrative")) { 0329 relation.isAdministrativeBoundary = true; 0330 } else if (key == QLatin1String("admin_level")) { 0331 relation.adminLevel = value.toInt(); 0332 } else { 0333 if ( shouldSave( Marble::WayType, key, value ) ) { 0334 way.save = true; 0335 } 0336 setCategory( way, key, value ); 0337 } 0338 } 0339 0340 long long lastRef = 0; 0341 for ( int i = 0; i < inputWay.refs_size(); i++ ) { 0342 lastRef += inputWay.refs( i ); 0343 way.nodes.push_back( lastRef ); 0344 } 0345 0346 if ( relation.isAdministrativeBoundary && !way.name.isEmpty() ) { 0347 relation.name = way.name; 0348 relation.ways << QPair<int, Marble::RelationRole>( inputWay.id(), Marble::Outer ); 0349 m_relations[inputWay.id()] = relation; 0350 } 0351 0352 if ( way.save || m_referencedWays.contains( inputWay.id() ) ) { 0353 if ( !way.isBuilding && way.nodes.size() > 1 && !m_referencedWays.contains( inputWay.id() ) ) { 0354 QList<int> nodes = way.nodes; 0355 way.nodes.clear(); 0356 way.nodes << nodes.first(); 0357 if ( nodes.size() > 2 ) { 0358 way.nodes << nodes.at( nodes.size() / 2 ); 0359 } 0360 way.nodes << nodes.last(); 0361 } 0362 0363 for( int node: way.nodes ) { 0364 m_referencedNodes << node; 0365 } 0366 0367 m_ways[inputWay.id()] = way; 0368 } 0369 } 0370 0371 m_currentEntity++; 0372 if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).ways_size() ) { 0373 m_currentEntity = 0; 0374 m_currentGroup++; 0375 if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() ) 0376 m_loadBlock = true; 0377 else 0378 loadGroup(); 0379 } 0380 } 0381 0382 void PbfParser::parseRelation() 0383 { 0384 if ( m_pass == 0 ) { 0385 const Relation& inputRelation = m_primitiveBlock.primitivegroup( m_currentGroup ).relations( m_currentEntity ); 0386 Marble::Relation relation; 0387 0388 for ( int tag = 0; tag < inputRelation.keys_size(); tag++ ) { 0389 0390 QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputRelation.keys( tag ) ).data() ); 0391 QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( inputRelation.vals( tag ) ).data() ); 0392 0393 if (key == QLatin1String("boundary") && value == QLatin1String("administrative")) { 0394 relation.isAdministrativeBoundary = true; 0395 } else if (key == QLatin1String("admin_level")) { 0396 relation.adminLevel = value.toInt(); 0397 } else if (key == QLatin1String("name")) { 0398 relation.name = value.trimmed(); 0399 } else if (key == QLatin1String("type") && value == QLatin1String("multipolygon")) { 0400 relation.isMultipolygon = true; 0401 } 0402 } 0403 0404 if ( relation.isAdministrativeBoundary ) { 0405 long long lastRef = 0; 0406 for ( int i = 0; i < inputRelation.types_size(); i++ ) { 0407 lastRef += inputRelation.memids( i ); 0408 switch ( inputRelation.types( i ) ) { 0409 case OSMPBF::Relation::NODE: 0410 relation.nodes.push_back( lastRef ); 0411 break; 0412 case OSMPBF::Relation::WAY: { 0413 string role = m_primitiveBlock.stringtable().s( inputRelation.roles_sid( i ) ).data(); 0414 Marble::RelationRole relationRole = Marble::None; 0415 if ( role == "outer" ) relationRole = Marble::Outer; 0416 if ( role == "inner" ) relationRole = Marble::Inner; 0417 m_referencedWays << lastRef; 0418 relation.ways.push_back( QPair<int, Marble::RelationRole>( lastRef, relationRole ) ); 0419 } 0420 break; 0421 case OSMPBF::Relation::RELATION: 0422 relation.relations.push_back( lastRef ); 0423 } 0424 } 0425 0426 m_relations[inputRelation.id()] = relation; 0427 } 0428 } 0429 0430 m_currentEntity++; 0431 if ( m_currentEntity >= m_primitiveBlock.primitivegroup( m_currentGroup ).relations_size() ) { 0432 m_currentEntity = 0; 0433 m_currentGroup++; 0434 if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() ) 0435 m_loadBlock = true; 0436 else 0437 loadGroup(); 0438 } 0439 } 0440 0441 void PbfParser::parseDense() 0442 { 0443 const DenseNodes& dense = m_primitiveBlock.primitivegroup( m_currentGroup ).dense(); 0444 if ( m_pass == 2 ) { 0445 m_lastDenseID += dense.id( m_currentEntity ); 0446 m_lastDenseLatitude += dense.lat( m_currentEntity ); 0447 m_lastDenseLongitude += dense.lon( m_currentEntity ); 0448 0449 Marble::Node node; 0450 node.lat = ( ( double ) m_lastDenseLatitude * m_primitiveBlock.granularity() + m_primitiveBlock.lat_offset() ) / ( 1000.0 * 1000.0 * 1000.0 ); 0451 node.lon = ( ( double ) m_lastDenseLongitude * m_primitiveBlock.granularity() + m_primitiveBlock.lon_offset() ) / ( 1000.0 * 1000.0 * 1000.0 ); 0452 0453 while ( true ) { 0454 if ( m_lastDenseTag >= dense.keys_vals_size() ) 0455 break; 0456 0457 int tagValue = dense.keys_vals( m_lastDenseTag ); 0458 if ( tagValue == 0 ) { 0459 m_lastDenseTag++; 0460 break; 0461 } 0462 0463 QString key = QString::fromUtf8( m_primitiveBlock.stringtable().s( dense.keys_vals( m_lastDenseTag ) ).data() ); 0464 QString value = QString::fromUtf8( m_primitiveBlock.stringtable().s( dense.keys_vals( m_lastDenseTag + 1 ) ).data() ); 0465 0466 if (key == QLatin1String("name")) { 0467 node.name = value.trimmed(); 0468 } else if (key == QLatin1String("addr:street")) { 0469 node.street = value.trimmed(); 0470 node.save = true; 0471 } else if (key == QLatin1String("addr:housenumber")) { 0472 node.houseNumber = value; 0473 node.save = true; 0474 } else if (key == QLatin1String("addr:city")) { 0475 node.city = value; 0476 node.save = true; 0477 } else { 0478 if ( shouldSave( Marble::NodeType, key, value ) ) { 0479 node.save = true; 0480 } 0481 setCategory( node, key, value ); 0482 } 0483 0484 m_lastDenseTag += 2; 0485 } 0486 0487 if ( node.save) { 0488 m_nodes[m_lastDenseID] = node; 0489 } 0490 0491 if ( m_referencedNodes.contains( m_lastDenseID ) ) { 0492 m_coordinates[m_lastDenseID] = node; 0493 } 0494 } 0495 0496 ++m_currentEntity; 0497 if ( m_currentEntity >= dense.id_size() ) { 0498 m_currentEntity = 0; 0499 m_currentGroup++; 0500 if ( m_currentGroup >= m_primitiveBlock.primitivegroup_size() ) { 0501 m_loadBlock = true; 0502 } else { 0503 loadGroup(); 0504 } 0505 } 0506 }