File indexing completed on 2024-03-24 03:53:34

0001 // SPDX-License-Identifier: LGPL-2.1-or-later
0002 //
0003 // SPDX-FileCopyrightText: 2011 Dennis Nienhüser <nienhueser@kde.org>
0004 //
0005 
0006 #include "upload.h"
0007 
0008 #include "logger.h"
0009 
0010 #include <QDebug>
0011 #include <QProcess>
0012 #include <QDateTime>
0013 #include <QDir>
0014 #include <QTemporaryFile>
0015 #include <QUrl>
0016 
0017 Upload::Upload(QObject *parent) :
0018     QObject(parent), m_uploadFiles(true)
0019 {
0020     // nothing to do
0021 }
0022 
0023 void Upload::changeStatus(const Package &package, const QString &status, const QString &message)
0024 {
0025     Logger::instance().setStatus(package.region.id() + QLatin1Char('_') + package.transport,
0026                                  package.region.name() + QLatin1String(" (") + package.transport + QLatin1Char(')'), status, message);
0027 }
0028 
0029 void Upload::processQueue()
0030 {
0031     if (m_queue.isEmpty()) {
0032         return;
0033     }
0034 
0035     Package const package = m_queue.takeFirst();
0036 
0037     if (upload(package)) {
0038         QString const message = QString("File %1 (%2) successfully created and uploaded").arg(package.file.fileName()).arg(Region::fileSize(package.file));
0039         changeStatus( package, "finished", message);
0040     }
0041     deleteFile(package.file);
0042     processQueue();
0043 }
0044 
0045 bool Upload::upload(const Package &package)
0046 {
0047     if (!m_uploadFiles) {
0048         return true;
0049     }
0050 
0051     QProcess ssh;
0052     QStringList arguments;
0053     QString const auth = "marble@filesmaster.kde.org";
0054     arguments << auth;
0055     arguments << "mkdir" << "-p";
0056     QString remoteDir = QString("/home/marble/web/monav/") + targetDir();
0057     arguments << remoteDir;
0058     ssh.start("ssh", arguments);
0059     ssh.waitForFinished(1000 * 60 * 10); // wait up to 10 minutes for mkdir to complete
0060     if (ssh.exitStatus() != QProcess::NormalExit || ssh.exitCode() != 0) {
0061         qDebug() << "Failed to create remote directory " << remoteDir;
0062         changeStatus(package, "error", QLatin1String("Failed to create remote directory: ") + ssh.readAllStandardError());
0063         return false;
0064     }
0065 
0066     QProcess scp;
0067     arguments.clear();
0068     arguments << package.file.absoluteFilePath();
0069     const QString target = remoteDir + QLatin1Char('/') + package.file.fileName();
0070     arguments << auth + QLatin1Char(':') + target;
0071     scp.start("scp", arguments);
0072     scp.waitForFinished(1000 * 60 * 60 * 12); // wait up to 12 hours for upload to complete
0073     if (scp.exitStatus() != QProcess::NormalExit || scp.exitCode() != 0) {
0074         qDebug() << "Failed to upload " << target;
0075         changeStatus(package, "error", QLatin1String("Failed to upload file: ") + scp.readAllStandardError());
0076         return false;
0077     }
0078 
0079     return adjustNewstuffFile(package);
0080 }
0081 
0082 void Upload::deleteFile(const QFileInfo &file)
0083 {
0084     if (!m_jobParameters.cacheData()) {
0085         QFile::remove(file.absoluteFilePath());
0086     }
0087 }
0088 
0089 bool Upload::adjustNewstuffFile(const Package &package)
0090 {
0091     if (m_xml.isNull()) {
0092         QTemporaryFile tempFile(QDir::tempPath() + QLatin1String("/monav-maps-XXXXXX.xml"));
0093         tempFile.setAutoRemove(false);
0094         tempFile.open();
0095         QString monavFilename = tempFile.fileName();
0096         tempFile.close();
0097         QProcess wget;
0098         QStringList const arguments = QStringList() << "http://filesmaster.kde.org/marble/newstuff/maps-monav.xml" << "-O" << monavFilename;
0099         wget.start("wget", arguments);
0100         wget.waitForFinished(1000 * 60 * 60 * 12); // wait up to 12 hours for download to complete
0101         if (wget.exitStatus() != QProcess::NormalExit || wget.exitCode() != 0) {
0102             qDebug() << "Failed to download newstuff file from filesmaster.kde.org";
0103             changeStatus( package, "error", QLatin1String("Failed to sync newstuff file: ") + wget.readAllStandardError());
0104             return false;
0105         }
0106 
0107         QFile file(monavFilename);
0108         if (!file.open(QFile::ReadOnly)) {
0109             qDebug() << "Failed to open newstuff file" << monavFilename;
0110             changeStatus( package, "error", "Failed to open newstuff file.");
0111             return false;
0112         }
0113 
0114         if ( !m_xml.setContent( &file ) ) {
0115             qDebug() << "Cannot parse newstuff xml file.";
0116             changeStatus( package, "error", "Failed to parse newstuff .xml file.");
0117             return false;
0118         }
0119 
0120         QFile::remove(monavFilename);
0121     }
0122 
0123     QDomElement root = m_xml.documentElement();
0124     QDomNodeList regions = root.elementsByTagName( "stuff" );
0125     for ( int i = 0; i < int(regions.length()); ++i ) {
0126         QDomNode node = regions.item( i );
0127         if (!node.namedItem("payload").isNull()) {
0128             QUrl url(node.namedItem("payload").toElement().text());
0129             QFileInfo fileInfo(url.path());
0130             if (fileInfo.fileName() == package.file.fileName()) {
0131                 QString removeFile;
0132                 QDomNode dateNode = node.namedItem("releasedate");
0133                 if (!dateNode.isNull()) {
0134                     dateNode.removeChild(dateNode.firstChild());
0135                     dateNode.appendChild(m_xml.createTextNode(releaseDate()));
0136                 }
0137                 QDomNode versionNode = node.namedItem("version");
0138                 if (!versionNode.isNull()) {
0139                     double version = versionNode.toElement().text().toDouble();
0140                     versionNode.removeChild(versionNode.firstChild());
0141                     versionNode.appendChild(m_xml.createTextNode(QString::number(version+0.1, 'f', 1)));
0142                 }
0143                 QDomNode payloadNode = node.namedItem("payload");
0144                 payloadNode.removeChild(payloadNode.firstChild());
0145                 if (fileInfo.dir().dirName() != targetDir()) {
0146                     removeFile = QString("/home/marble/web/monav/%1/%2").arg(fileInfo.dir().dirName()).arg(package.file.fileName());
0147                     qDebug() << "Going to remove the old file " << removeFile;
0148                 }
0149                 QString payload = "http://files.kde.org/marble/monav/%1/%2";
0150                 payload = payload.arg(targetDir()).arg(package.file.fileName());
0151                 payloadNode.appendChild(m_xml.createTextNode(payload));
0152                 return removeFile.isEmpty() ? uploadNewstuff() : (uploadNewstuff() && deleteRemoteFile(removeFile));
0153             }
0154         }
0155     }
0156 
0157     QDomNode stuff = root.appendChild(m_xml.createElement("stuff"));
0158     stuff.toElement().setAttribute("category", "marble/routing/monav");
0159     QDomNode nameNode = stuff.appendChild(m_xml.createElement("name"));
0160     nameNode.toElement().setAttribute("lang", "en");
0161     QString name = "%1 / %2 (%3)";
0162     if (package.region.country().isEmpty()) {
0163         name = name.arg(package.region.continent()).arg(package.region.name());
0164         name = name.arg(package.transport);
0165     } else {
0166         name = "%1 / %2 / %3 (%4)";
0167         name = name.arg(package.region.continent()).arg(package.region.country());
0168         name = name.arg(package.region.name()).arg(package.transport);
0169     }
0170     nameNode.appendChild(m_xml.createTextNode(name));
0171 
0172     QDomNode authorNode = stuff.appendChild(m_xml.createElement("author"));
0173     authorNode.appendChild(m_xml.createTextNode("Automatically created from map data assembled by the OpenStreetMap community"));
0174 
0175     QDomNode licenseNode = stuff.appendChild(m_xml.createElement("license"));
0176     licenseNode.appendChild(m_xml.createTextNode("Creative Commons by-SA 2.0"));
0177 
0178     QDomNode summaryNode = stuff.appendChild(m_xml.createElement("summary"));
0179     QString summary = "Requires KDE >= 4.6: Offline Routing in %1, %2";
0180     summary = summary.arg(package.region.name()).arg(package.region.continent());
0181     summaryNode.appendChild(m_xml.createTextNode(summary));
0182 
0183     QDomNode versionNode = stuff.appendChild(m_xml.createElement("version"));
0184     versionNode.appendChild(m_xml.createTextNode("0.1"));
0185 
0186     QDomNode dateNode = stuff.appendChild(m_xml.createElement("releasedate"));
0187     dateNode.appendChild(m_xml.createTextNode(releaseDate()));
0188 
0189     QDomNode previewNode = stuff.appendChild(m_xml.createElement("preview"));
0190     QString preview = "http://files.kde.org/marble/monav/previews/%1-preview.png";
0191     preview = preview.arg(package.region.id());
0192     previewNode.appendChild(m_xml.createTextNode(preview));
0193 
0194     QDomNode payloadNode = stuff.appendChild(m_xml.createElement("payload"));
0195     payloadNode.toElement().setAttribute("lang", "en");
0196     QString payload = "http://files.kde.org/marble/monav/%1/%2";
0197     payload = payload.arg(targetDir()).arg(package.file.fileName());
0198     payloadNode.appendChild(m_xml.createTextNode(payload));
0199 
0200     return uploadNewstuff();
0201 }
0202 
0203 bool Upload::uploadNewstuff()
0204 {
0205     QTemporaryFile outFile(QDir::tempPath() + QLatin1String("/monav-maps-out-XXXXXX.xml"));
0206     outFile.open();
0207     QTextStream outStream(&outFile);
0208     outStream << m_xml.toString(2);
0209     outStream.flush();
0210 
0211     QProcess scp;
0212     QStringList arguments;
0213     arguments << outFile.fileName();
0214     arguments << "marble@filesmaster.kde.org:/home/marble/web/newstuff/maps-monav.xml";
0215     scp.start("scp", arguments);
0216     scp.waitForFinished(1000 * 60 * 60 * 12); // wait up to 12 hours for upload to complete
0217     if (scp.exitStatus() != QProcess::NormalExit || scp.exitCode() != 0) {
0218         qDebug() << "Failed to upload " << outFile.fileName() << ": " << scp.readAllStandardError();
0219         return false;
0220     }
0221 
0222     return true;
0223 }
0224 
0225 bool Upload::deleteRemoteFile(const QString &filename)
0226 {
0227     if (filename.isEmpty()) {
0228         return true;
0229     }
0230 
0231     if (!filename.startsWith(QLatin1String( "/home/marble/" ))) {
0232         return false;
0233     }
0234 
0235     QProcess ssh;
0236     QStringList arguments;
0237     arguments << "marble@filesmaster.kde.org" << "rm" << filename;
0238     ssh.start("ssh", arguments);
0239     ssh.waitForFinished(1000 * 60 * 10); // wait up to 10 minutes for rm to complete
0240     if (ssh.exitStatus() != QProcess::NormalExit || ssh.exitCode() != 0) {
0241         qDebug() << "Failed to delete remote file " << filename;
0242         return false;
0243     }
0244 
0245     return true;
0246 }
0247 
0248 void Upload::uploadAndDelete(const Region &region, const QFileInfo &file, const QString &transport)
0249 {
0250     Package package;
0251     package.region = region;
0252     package.file = file;
0253     package.transport = transport;
0254 
0255     m_queue.removeAll(package);
0256     m_queue << package;
0257     processQueue();
0258 }
0259 
0260 bool Upload::Package::operator ==(const Upload::Package &other) const
0261 {
0262     return region == other.region;
0263 }
0264 
0265 Upload &Upload::instance()
0266 {
0267     static Upload m_instance;
0268     return m_instance;
0269 }
0270 
0271 bool Upload::uploadFiles() const
0272 {
0273     return m_uploadFiles;
0274 }
0275 
0276 void Upload::setJobParameters(const JobParameters &parameters)
0277 {
0278     m_jobParameters = parameters;
0279 }
0280 
0281 void Upload::setUploadFiles(bool arg)
0282 {
0283     m_uploadFiles = arg;
0284 }
0285 
0286 QString Upload::targetDir() const
0287 {
0288     QString targetDir = "%1-w%2";
0289     targetDir = targetDir.arg(QDateTime::currentDateTime().date().year());
0290     targetDir = targetDir.arg(QDateTime::currentDateTime().date().weekNumber());
0291     return targetDir;
0292 }
0293 
0294 QString Upload::releaseDate() const
0295 {
0296     return QDateTime::currentDateTime().toString("MM/dd/yy");
0297 }
0298 
0299 #include "moc_upload.cpp"