lib/backends/hafasqueryparser.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2018 Volker Krause <vkrause@kde.org>
0003
0004     SPDX-License-Identifier: LGPL-2.0-or-later
0005 */
0006
0007 #include "hafasqueryparser.h"
0008 #include "hafasjourneyresponse_p.h"
0009 #include "logging.h"
0010
0011 #include <json/jsonp_p.h>
0012
0013 #include <KPublicTransport/Journey>
0014 #include <KPublicTransport/Location>
0015 #include <KPublicTransport/Stopover>
0016
0017 #include <QDateTime>
0018 #include <QDebug>
0019 #include <QJsonArray>
0020 #include <QJsonDocument>
0021 #include <QJsonObject>
0022 #include <QRegularExpression>
0023 #include <QXmlStreamReader>
0024
0025 #include <zlib.h>
0026
0027 using namespace KPublicTransport;
0028
0029 HafasQueryParser::HafasQueryParser() = default;
0030 HafasQueryParser::~HafasQueryParser() = default;
0031
0032 std::vector<Stopover> HafasQueryParser::parseStationBoardResponse(const QByteArray &data, bool isArrival)
0033 {
0034     clearErrorState();
0035     qDebug().noquote() << data;
0036     std::vector<Stopover> res;
0037
0038     QXmlStreamReader reader;
0039     if (data.startsWith("<Journey")) { // SBB and RT don't reply with valid XML...
0040         reader.addData("<dummyRoot>");
0041         reader.addData(data);
0042         reader.addData("</dummyRoot>");
0043     } else {
0044         reader.addData(data);
0045     }
0046
0047     Location stopPoint;
0048     while (!reader.atEnd()) {
0049         const auto token = reader.readNext();
0050         switch (token) {
0051             case QXmlStreamReader::StartElement:
0052                 if (reader.name() == QLatin1String("St")) {
0053                     stopPoint.setName(reader.attributes().value(QLatin1String("name")).toString());
0054                     setLocationIdentifier(stopPoint, reader.attributes().value(QLatin1String("evaId")).toString());
0055                 } else if (reader.name() == QLatin1String("Journey")) {
0056                     auto dt = QDateTime::fromString(reader.attributes().value(QLatin1String("fpDate")) + reader.attributes().value(QLatin1String("fpTime")), QStringLiteral("dd.MM.yyhh:mm"));
0057                     if (dt.date().year() < 2000) {
0058                         dt = dt.addYears(100);
0059                     }
0060                     const auto delayStr = reader.attributes().value(QLatin1String("e_delay"));
0061                     const auto delaySecs = delayStr.toInt() * 60;
0062                     Stopover dep;
0063                     if (isArrival) {
0064                         dep.setScheduledArrivalTime(dt);
0065                         if (!delayStr.isEmpty()) {
0066                             dep.setExpectedArrivalTime(dt.addSecs(delaySecs));
0067                         }
0068                     } else {
0069                         dep.setScheduledDepartureTime(dt);
0070                         if (!delayStr.isEmpty()) {
0071                             dep.setExpectedDepartureTime(dt.addSecs(delaySecs));
0072                         }
0073                     }
0074                     dep.setScheduledPlatform(reader.attributes().value(QLatin1String("platform")).toString());
0075                     dep.setExpectedPlatform(reader.attributes().value(QLatin1String("newpl")).toString());
0076
0077                     if (reader.attributes().value(QLatin1String("delay")) == QLatin1String("cancel")) {
0078                         dep.setDisruptionEffect(Disruption::NoService);
0079                     }
0080
0081                     Route route;
0082                     route.setDirection(reader.attributes().value(QLatin1String("targetLoc")).toString());
0083                     Line line;
0084                     line.setName(reader.attributes().value(QLatin1String("hafasname")).toString());
0085                     if (line.name().isEmpty()) {
0086                         const auto prod = reader.attributes().value(QLatin1String("prod"));
0087                         const auto idx = prod.indexOf(QLatin1Char('#'));
0088                         line.setName(prod.left(idx).toString().simplified());
0089                     }
0090
0091                     line.setMode(parseLineMode(reader.attributes().value(QLatin1String("class"))));
0092                     // TODO line mode from second part of prod attribute, if class not set
0093                     route.setLine(line);
0094                     dep.setRoute(route);
0095
0096                     dep.setStopPoint(stopPoint);
0097                     res.push_back(dep);
0098                 }
0099                 break;
0100             default:
0101                 break;
0102         }
0103     }
0104
0105     return res;
0106 }
0107
0108 std::vector<Location> HafasQueryParser::parseGetStopResponse(const QByteArray &data)
0109 {
0110     clearErrorState();
0111
0112     QJsonParseError parseError;
0113     const auto doc = QJsonDocument::fromJson(JsonP::decode(data), &parseError);
0114     if (parseError.error != QJsonParseError::NoError) {
0115         qCWarning(Log) << parseError.errorString() << data;
0116     }
0117     const auto suggestions = doc.object().value(QLatin1String("suggestions")).toArray();
0118     std::vector<Location> res;
0119     res.reserve(suggestions.size());
0120     for (const auto &suggestion : suggestions) {
0121         const auto obj = suggestion.toObject();
0122         const auto extId = obj.value(QLatin1String("extId")).toString();
0123         if (extId.isEmpty()) {
0124             continue; // not a stop/station
0125         }
0126         Location loc;
0127         setLocationIdentifier(loc, extId);
0128         loc.setName(obj.value(QLatin1String("value")).toString());
0129         loc.setLatitude(obj.value(QLatin1String("ycoord")).toString().toInt() / 1000000.0);
0130         loc.setLongitude(obj.value(QLatin1String("xcoord")).toString().toInt() / 1000000.0);
0131         res.push_back(loc);
0132     }
0133
0134     return res;
0135 }
0136
0137 std::vector<Location> HafasQueryParser::parseQueryLocationResponse(const QByteArray &data)
0138 {
0139     clearErrorState();
0140
0141     QJsonParseError parseError;
0142     auto doc = QJsonDocument::fromJson(data, &parseError);
0143     if (parseError.error != QJsonParseError::NoError) {
0144         qCWarning(Log) << parseError.errorString() << data;
0145
0146         // try again after attempting to fix SBB's creative JSON
0147         auto s = QString::fromUtf8(data);
0148         s.replace(QRegularExpression(QStringLiteral("([a-zI]+)\\s*:")), QStringLiteral("\"\\1\":"));
0149         doc = QJsonDocument::fromJson(s.toUtf8(), &parseError);
0150         qDebug() << parseError.errorString();
0151     }
0152     //qDebug().noquote() << doc.toJson();
0153     const auto stops = doc.object().value(QLatin1String("stops")).toArray();
0154     std::vector<Location> res;
0155     res.reserve(stops.size());
0156     for (const auto &stop : stops) {
0157         const auto obj = stop.toObject();
0158         Location loc;
0159         setLocationIdentifier(loc, obj.value(QLatin1String("extId")).toString());
0160         loc.setName(obj.value(QLatin1String("name")).toString());
0161         loc.setLatitude(obj.value(QLatin1String("y")).toString().toInt() / 1000000.0);
0162         loc.setLongitude(obj.value(QLatin1String("x")).toString().toInt() / 1000000.0);
0163         res.push_back(loc);
0164     }
0165     return res;
0166 }
0167
0168 static QByteArray gzipDecompress(const QByteArray &data)
0169 {
0170     QByteArray rawData;
0171     z_stream stream;
0172     unsigned char buffer[1024];
0173
0174     stream.zalloc = nullptr;
0175     stream.zfree = nullptr;
0176     stream.opaque = nullptr;
0177     stream.avail_in = data.size();
0178     stream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(data.data()));
0179
0180     auto ret = inflateInit2(&stream, 15 + 32); // see docs, the magic numbers enable gzip decoding
0181     if (ret != Z_OK) {
0182         qCWarning(Log) << "Failed to initialize zlib stream.";
0183         return {};
0184     }
0185
0186     do {
0187         stream.avail_out = sizeof(buffer);
0188         stream.next_out = buffer;
0189
0190         ret = inflate(&stream, Z_NO_FLUSH);
0191         if (ret != Z_OK && ret != Z_STREAM_END) {
0192             qCWarning(Log) << "Zlib decoding failed!" << ret;
0193             break;
0194         }
0195
0196         rawData.append(reinterpret_cast<char*>(buffer), sizeof(buffer) - stream.avail_out);
0197     } while (stream.avail_out == 0);
0198     inflateEnd(&stream);
0199
0200     return rawData;
0201 }
0202
0203 static QDateTime parseDateTime(const QDate &baseDate, uint16_t time)
0204 {
0205     if (time == 0xffff) { // value is unset
0206         return {};
0207     }
0208
0209     const auto days = time / 2400;
0210     const auto hours = (time / 100) % 24;
0211     const auto mins = time % 100;
0212
0213     auto dt = QDateTime(baseDate, QTime(hours, mins));
0214     return dt.addDays(days);
0215 }
0216
0217 std::vector<Journey> HafasQueryParser::parseQueryJourneyResponse(const QByteArray &data)
0218 {
0219 #if Q_BYTE_ORDER == Q_BIG_ENDIAN
0220 #warning Hafas binary response parsing not implemented on big endian architectures!
0221     Q_UNUSED(data);
0222     return {};
0223 #endif
0224     clearErrorState();
0225
0226     // yes, this is gzip compressed rather than using the HTTP compression transparently...
0227     const auto rawData = gzipDecompress(data);
0228
0229     if (rawData.size() < (int)sizeof(HafasJourneyResponseHeader)) {
0230         qCWarning(Log) << "Response too small for header structure.";
0231         return {};
0232     }
0233     const auto header = reinterpret_cast<const HafasJourneyResponseHeader*>(rawData.constData());
0234     qDebug() << header->version << header->numJourneys;
0235
0236     if (rawData.size() < (int)(sizeof(HafasJourneyResponseExtendedHeader) + header->extendedHeaderOffset)) {
0237         qCWarning(Log) << "Response too short for extended header structure.";
0238         return {};
0239     }
0240     const auto extHeader = reinterpret_cast<const HafasJourneyResponseExtendedHeader*>(rawData.constData() + header->extendedHeaderOffset);
0241     if (extHeader->length < (int)sizeof(HafasJourneyResponseExtendedHeader)) {
0242         qCWarning(Log) << "Extended header is shorter than expected" << extHeader->length;
0243         return {};
0244     }
0245     if (extHeader->errorCode != 0) {
0246         qCDebug(Log) << "Journey query returned error" << extHeader->errorCode;
0247         // TODO we could distinguish between not found and service errors here
0248         m_error = Reply::NotFoundError;
0249         return {};
0250     }
0251
0252     const auto detailsHeader = reinterpret_cast<const HafasJourneyResponseDetailsHeader*>(rawData.constData() + extHeader->detailsOffset);
0253     qDebug() << "details header:" << detailsHeader->version;
0254
0255     HafasJourneyResponseStringTable stringTable(rawData, header->stringTableOffset, extHeader->encodingStr);
0256     QDate baseDate(1979, 12, 31);
0257     baseDate = baseDate.addDays(header->date);
0258
0259     std::vector<Journey> journeys;
0260     journeys.reserve(header->numJourneys);
0261     for (int journeyIdx = 0; journeyIdx < header->numJourneys; ++journeyIdx) {
0262         const auto journeyInfo = reinterpret_cast<const HafasJourneyResponseJourney*>(rawData.constData() + sizeof(HafasJourneyResponseHeader) + journeyIdx * sizeof(HafasJourneyResponseJourney));
0263         qDebug() << "section count: " << journeyInfo->numSections;
0264
0265         const auto journeyDetailsOffset = *reinterpret_cast<const uint16_t*>(rawData.constData()
0266             + extHeader->detailsOffset
0267             + detailsHeader->journeyDetailsIndexOffset
0268             + journeyIdx * sizeof(uint16_t));
0269
0270         std::vector<JourneySection> sections;
0271         sections.reserve(journeyInfo->numSections);
0272         for (int sectionIdx = 0; sectionIdx < journeyInfo->numSections; ++sectionIdx) {
0273             const auto sectionInfo = reinterpret_cast<const HafasJourneyResponseSection*>(rawData.constData()
0274                 + sizeof(HafasJourneyResponseHeader)
0275                 + journeyInfo->sectionsOffset
0276                 + sectionIdx * sizeof(HafasJourneyResponseSection));
0277             qDebug() << stringTable.lookup(sectionInfo->lineNameStr) << sectionInfo->type;
0278
0279             Location from;
0280             const auto fromInfo = reinterpret_cast<const HafasJourneyResponseStation*>(rawData.constData()
0281                 + header->stationTableOffset
0282                 + sectionInfo->departureStationIdx * sizeof(HafasJourneyResponseStation));
0283             from.setName(stringTable.lookup(fromInfo->nameStr));
0284             from.setLatitude(fromInfo->latitude / 1000000.0);
0285             from.setLongitude(fromInfo->longitude / 1000000.0);
0286             setLocationIdentifier(from, QString::number(fromInfo->id));
0287
0288             Location to;
0289             const auto toInfo = reinterpret_cast<const HafasJourneyResponseStation*>(rawData.constData()
0290                 + header->stationTableOffset
0291                 + sectionInfo->arrivalStationIdx * sizeof(HafasJourneyResponseStation));
0292             to.setName(stringTable.lookup(toInfo->nameStr));
0293             to.setLatitude(toInfo->latitude / 1000000.0);
0294             to.setLongitude(toInfo->longitude / 1000000.0);
0295             setLocationIdentifier(to, QString::number(toInfo->id));
0296
0297             JourneySection section;
0298             section.setFrom(from);
0299             section.setTo(to);
0300
0301             section.setScheduledDepartureTime(parseDateTime(baseDate, sectionInfo->scheduledDepartureTime));
0302             section.setScheduledArrivalTime(parseDateTime(baseDate, sectionInfo->scheduledArrivalTime));
0303
0304             const auto journeyAttrIndex = *reinterpret_cast<const uint16_t*>(rawData.constData()
0305                 + extHeader->journeyAttributesIndexOffset + journeyIdx * sizeof(uint16_t));
0306             auto attr = HafasJourneyResponse::attribute(rawData.constData(), extHeader, journeyAttrIndex);
0307             while (!attr->atEnd()) {
0308                 qDebug() << "journey attr" << stringTable.lookup(attr->keyStr) << stringTable.lookup(attr->valueStr);
0309                 ++attr;
0310             }
0311
0312             if (sectionInfo->type == HafasJourneyResponseSectionMode::PublicTransport) {
0313                 Route route;
0314                 Line line;
0315                 line.setName(stringTable.lookup(sectionInfo->lineNameStr).trimmed());
0316
0317                 auto attr = HafasJourneyResponse::attribute(rawData.constData(), extHeader, sectionInfo->sectionAttributeIndex);
0318                 while (!attr->atEnd()) {
0319                     qDebug() << "section attr" << stringTable.lookup(attr->keyStr) << stringTable.lookup(attr->valueStr);
0320                     const auto key = stringTable.lookup(attr->keyStr);
0321                     if (key == QLatin1String("Direction")) {
0322                         const auto dir = stringTable.lookup(attr->valueStr);
0323                         if (dir != QLatin1String("---")) {
0324                             route.setDirection(dir.trimmed());
0325                         }
0326                     } else if (key == QLatin1String("Class")) {
0327                         line.setMode(parseLineMode(stringTable.lookup(attr->valueStr)));
0328                     }
0329                     ++attr;
0330                 }
0331
0332                 const auto *commentPtr = reinterpret_cast<const uint16_t*>(rawData.constBegin() + header->commentTableOffset + sectionInfo->commentIdx);
0333                 const auto commentCount = *commentPtr;
0334                 for (int i = 0; i < commentCount; ++i) {
0335                     ++commentPtr;
0336                     // format: XX - <human readable comment>, where XX is two character code for the comment
0337                     const auto note = stringTable.lookup(*commentPtr);
0338                     if (note.size() > 5 && QStringView(note).mid(2, 3) == QLatin1String(" - ")) {
0339                         section.addNote(note.mid(5));
0340                     } else {
0341                         section.addNote(note);
0342                     }
0343                 }
0344
0345                 route.setLine(line);
0346                 section.setRoute(route);
0347                 section.setMode(JourneySection::PublicTransport);
0348                 section.setScheduledDeparturePlatform(stringTable.lookup(sectionInfo->scheduledDeparturePlatformStr));
0349                 section.setScheduledArrivalPlatform(stringTable.lookup(sectionInfo->scheduledArrivalPlatformStr));
0350
0351                 const auto sectionDetail = reinterpret_cast<const HafasJourneyResponseSectionDetail*>(rawData.constData()
0352                     + extHeader->detailsOffset
0353                     + journeyDetailsOffset
0354                     + detailsHeader->sectionDetailsOffset
0355                     + detailsHeader->sectionDetailsSize * sectionIdx);
0356                 qDebug() << "section detail" << sectionDetail->expectedArrivalPlatformStr <<  sectionDetail->expectedDeparturePlatformStr;
0357                 section.setExpectedDeparturePlatform(stringTable.lookup(sectionDetail->expectedDeparturePlatformStr));
0358                 section.setExpectedArrivalPlatform(stringTable.lookup(sectionDetail->expectedArrivalPlatformStr));
0359
0360                 section.setExpectedDepartureTime(parseDateTime(baseDate, sectionDetail->expectedDepartureTime));
0361                 section.setExpectedArrivalTime(parseDateTime(baseDate, sectionDetail->expectedArrivalTime));
0362
0363                 std::vector<Stopover> stops;
0364                 stops.reserve(sectionDetail->numStops);
0365                 for (int i = 0; i < sectionDetail->numStops; ++i) {
0366                     const auto stopInfo = reinterpret_cast<const HafasJourneyResponseStop*>(rawData.constData() + extHeader->detailsOffset + detailsHeader->stopsOffset + i * detailsHeader->stopsSize);
0367
0368                     Location loc;
0369                     const auto locInfo = reinterpret_cast<const HafasJourneyResponseStation*>(rawData.constData()
0370                         + header->stationTableOffset
0371                         + stopInfo->stationIdx * sizeof(HafasJourneyResponseStation));
0372                     loc.setName(stringTable.lookup(locInfo->nameStr));
0373                     loc.setLatitude(locInfo->latitude / 1000000.0);
0374                     loc.setLongitude(locInfo->longitude / 1000000.0);
0375
0376                     Stopover stop;
0377                     stop.setStopPoint(loc);
0378                     stop.setScheduledArrivalTime(parseDateTime(baseDate, stopInfo->scheduledArrivalTime));
0379                     stop.setScheduledDepartureTime(parseDateTime(baseDate, stopInfo->scheduledDepartureTime));
0380                     stop.setExpectedArrivalTime(parseDateTime(baseDate, stopInfo->expectedArrivalTime));
0381                     stop.setExpectedDepartureTime(parseDateTime(baseDate, stopInfo->expectedDepartureTime));
0382                     stop.setScheduledPlatform(stringTable.lookup(stopInfo->scheduledDeparturePlatformStr));
0383                     stop.setExpectedPlatform(stringTable.lookup(stopInfo->expectedDeparturePlatformStr));
0384
0385                     stops.push_back(stop);
0386                 }
0387                 section.setIntermediateStops(std::move(stops));
0388
0389             } else if (sectionInfo->type == HafasJourneyResponseSectionMode::Walk) {
0390                 section.setMode(JourneySection::Walking);
0391             } else if (sectionInfo->type == HafasJourneyResponseSectionMode::Transfer1 || sectionInfo->type == HafasJourneyResponseSectionMode::Transfer2) {
0392                 section.setMode(JourneySection::Transfer);
0393             } else {
0394                 qCWarning(Log) << "Unknown section mode:" << sectionInfo->type;
0395             }
0396
0397             sections.push_back(section);
0398         }
0399
0400         const auto disruptionTable = HafasJourneyResponse::disruptionTable(rawData.constData(), extHeader);
0401         auto disruption = disruptionTable->firstDisruptionForJourney(journeyIdx);
0402         while (disruption) {
0403             qDebug() << "disruption" << stringTable.lookup(disruption->idStr) << stringTable.lookup(disruption->titleStr) << stringTable.lookup(disruption->messageStr)
0404                 << stringTable.lookup(disruption->startStr) << stringTable.lookup(disruption->endStr) << disruption->bitmask << disruption->section;
0405             sections[disruption->section].addNote(stringTable.lookup(disruption->titleStr));
0406             sections[disruption->section].addNote(stringTable.lookup(disruption->messageStr));
0407
0408             auto attr = HafasJourneyResponse::attribute(rawData.constData(), extHeader, disruption->disruptionAttributeIndex);
0409             while (!attr->atEnd()) {
0410                 qDebug() << "disruption attr" << stringTable.lookup(attr->keyStr) << stringTable.lookup(attr->valueStr);
0411                 ++attr;
0412             }
0413
0414             disruption = disruption->next(disruptionTable);
0415         }
0416
0417         Journey journey;
0418         journey.setSections(std::move(sections));
0419         journeys.push_back(journey);
0420     }
0421
0422     return journeys;
0423 }