File indexing completed on 2024-12-01 07:25:51

 
 /*
     SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
 
     SPDX-License-Identifier: LGPL-2.0-or-later
 */
 
 #include <gbfs/gbfsjob.h>
 #include <gbfs/gbfsservice.h>
 
 #include <QCoreApplication>
 #include <QDebug>
 #include <QDirIterator>
 #include <QJsonArray>
 #include <QJsonDocument>
 #include <QJsonObject>
 #include <QNetworkAccessManager>
 #include <QNetworkReply>
 #include <QNetworkRequest>
 #include <QRectF>
 #include <QTimer>
 
 #include <iostream>
 #include <random>
 
 using namespace KPublicTransport;
 
 /** Determine bounding boxes of the initial set of GBFS feeds we ship. */
 class GBFSProbe : public QObject
 {
     Q_OBJECT
 public:
     explicit GBFSProbe(QObject *parent = nullptr);
 
     void start();
     void getFeedList();
     void discoverNextFeed();
     void checkDuplicateSystemIds();
     void writeFeeds();
 
     QNetworkAccessManager m_nam;
     QStringList m_gbfsFeeds;
     int m_currentFeedIdx = -1;
     int m_throttleTime = 0;
     QStringList m_throttledFeeds;
     bool m_syntheticSystemId = false;
 
     std::vector<GBFSService> m_services;
     QString m_outputFileName;
 };
 
 GBFSProbe::GBFSProbe(QObject *parent)
     : QObject(parent)
 {
     m_nam.setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
 }
 
 void GBFSProbe::start()
 {
     getFeedList();
 }
 
 void GBFSProbe::getFeedList()
 {
     auto reply = m_nam.get(QNetworkRequest(QUrl(QStringLiteral("https://raw.githubusercontent.com/NABSA/gbfs/master/systems.csv"))));
     connect(reply, &QNetworkReply::finished, this, [this, reply]() {
         reply->deleteLater();
         if (reply->error() != QNetworkReply::NoError) {
             qCritical() << reply->errorString();
             QCoreApplication::exit(1);
             return;
         }
 
         reply->readLine(); // skip header line
         while (!reply->atEnd()) {
             auto line = reply->readLine();
             line.replace("http:", "https:");
             const auto idx = line.lastIndexOf("https");
             if (idx < 0) {
                 continue;
             }
             line = line.mid(idx).trimmed();
             if (line.endsWith('"')) {
                 line.chop(1);
             }
             m_gbfsFeeds.push_back(QString::fromUtf8(line));
         }
 
         std::sort(m_gbfsFeeds.begin(), m_gbfsFeeds.end());
         m_gbfsFeeds.erase(std::unique(m_gbfsFeeds.begin(), m_gbfsFeeds.end()), m_gbfsFeeds.end());
 
         QFile extraFeeds(QStringLiteral("gbfs-feeds.txt"));
         if (!extraFeeds.open(QFile::ReadOnly)) {
             qCritical() << extraFeeds.errorString();
             QCoreApplication::exit(1);
             return;
         }
         const auto lines = QString::fromUtf8(extraFeeds.readAll()).split(QLatin1Char('\n'));
         for (const auto &line : lines) {
             const auto extraFeed = line.trimmed();
             if (extraFeed.isEmpty()) {
                 continue;
             }
 
             const auto it = std::lower_bound(m_gbfsFeeds.begin(), m_gbfsFeeds.end(), extraFeed);
             if (it != m_gbfsFeeds.end() && (*it) == extraFeed) {
                 qDebug() << "Extra feed already in NABSA systems.csv:" << extraFeed;
                 continue;
             }
             m_gbfsFeeds.insert(it, extraFeed);
         }
 
         qDebug() << "Found" << m_gbfsFeeds.size() << "possible feeds - running discovery on them...";
         std::shuffle(m_gbfsFeeds.begin(), m_gbfsFeeds.end(), std::default_random_engine()); // reduce risk of spamming the same service with too many requests
         discoverNextFeed();
     });
 }
 
 void GBFSProbe::discoverNextFeed()
 {
     ++m_currentFeedIdx;
     if (m_currentFeedIdx >= m_gbfsFeeds.size()) {
         if (!m_throttledFeeds.isEmpty()) {
             qDebug() << "Retrying for" << m_throttledFeeds.size() << "throttled feeds";
             m_gbfsFeeds = std::move(m_throttledFeeds);
             std::shuffle(m_gbfsFeeds.begin(), m_gbfsFeeds.end(), std::default_random_engine());
             m_throttledFeeds.clear();
             m_currentFeedIdx = 0;
             m_throttleTime = m_throttleTime == 0 ? 500 : (2 * m_throttleTime);
         } else {
             checkDuplicateSystemIds();
             return;
         }
     }
 
     auto job = new GBFSJob(&m_nam);
     job->setRequestedData({GBFS::StationInformation, GBFS::FreeBikeStatus, GBFS::GeofencingZones}); // everything we can use for the bounding box
     QObject::connect(job, &GBFSJob::finished, this, [job, this]() {
         job->deleteLater();
         if (job->error() == GBFSJob::TooManyRequestsError) {
             m_throttledFeeds.push_back(m_gbfsFeeds[m_currentFeedIdx]);
             qWarning() << "Scheduling for later:" << m_gbfsFeeds[m_currentFeedIdx] << job->errorMessage();
         } else if (job->error() != GBFSJob::NoError) {
             qWarning() << m_gbfsFeeds[m_currentFeedIdx] << job->error() << job->errorMessage() << "- skipping";
         } else if (job->service().boundingBox.isEmpty()) {
             qWarning() << m_gbfsFeeds[m_currentFeedIdx] << "has an empty bounding box - skipping";
         } else {
             m_services.push_back(job->service());
         }
         QTimer::singleShot(m_throttleTime, this, &GBFSProbe::discoverNextFeed);
     });
 
     GBFSService service;
     service.discoveryUrl = QUrl(m_gbfsFeeds[m_currentFeedIdx]);
     if (m_syntheticSystemId) {
         service.generateSystemId();
     }
     job->discoverAndUpdate(service);
 }
 
 static bool sortBySystemId(const GBFSService &lhs, const GBFSService &rhs)
 {
     return lhs.systemId < rhs.systemId;
 }
 
 void GBFSProbe::checkDuplicateSystemIds()
 {
     m_gbfsFeeds.clear();
 
     std::sort(m_services.begin(), m_services.end(), sortBySystemId);
     for (auto it = m_services.begin(); it != m_services.end();) {
         const auto range = std::equal_range(it, m_services.end(), (*it), sortBySystemId);
         assert(range.first == it);
         if (std::next(it) == range.second) {
             it = range.second;
         } else {
             for (auto it2 = range.first; it2 != range.second; ++it2) {
                 m_gbfsFeeds.push_back((*it2).discoveryUrl.toString());
             }
             it = m_services.erase(range.first, range.second);
         }
     }
 
     qWarning() << "Feeds with colliding system ids:" << m_gbfsFeeds.size() << m_gbfsFeeds;
     if (!m_gbfsFeeds.isEmpty() && !m_syntheticSystemId) {
         m_syntheticSystemId = true;
         m_currentFeedIdx = -1;
         m_throttleTime = 0;
         std::shuffle(m_gbfsFeeds.begin(), m_gbfsFeeds.end(), std::default_random_engine());
         discoverNextFeed();
     } else {
         writeFeeds();
     }
 }
 
 void GBFSProbe::writeFeeds()
 {
     QJsonArray array;
     for (const auto &service : m_services) {
         array.push_back(GBFSService::toJson(service));
     };
 
     auto b = QJsonDocument(array).toJson(QJsonDocument::Compact);
     b.replace("},{", "},\n {");
     b.replace("[{", "[\n {");
     b.replace("}]", "}\n]\n");
 
     QFile f(m_outputFileName);
     if (!f.open(QFile::WriteOnly)) {
         qCritical() << f.errorString();
         QCoreApplication::exit(1);
         return;
     }
     f.write(b);
     f.close();
 
     qDebug() << m_services.size() << "valid feeds discovered";
     QCoreApplication::quit();
 }
 
 int main(int argc, char **argv)
 {
     QCoreApplication app(argc, argv);
     if (app.arguments().size() <= 1) {
         std::cerr << "Usage: " << argv[0] << " [path to GBFS services file]" << std::endl;
         return 1;
     }
 
     GBFSProbe probe;
     probe.m_outputFileName = app.arguments().at(1);
     QMetaObject::invokeMethod(&probe, &GBFSProbe::start, Qt::QueuedConnection);
 
     return app.exec();
 }
 
 #include "gbfsprobe.moc"