File indexing completed on 2024-02-18 04:58:15

0001 /* This file is part of the KDE project
0002 
0003    Copyright (C) 2008 Lukas Appelhans <l.appelhans@gmx.de>
0004    Copyright (C) 2009 Matthias Fuchs <mat69@gmx.net>
0005 
0006    This program is free software; you can redistribute it and/or
0007    modify it under the terms of the GNU General Public
0008    License as published by the Free Software Foundation; either
0009    version 2 of the License, or (at your option) any later version.
0010 */
0011 #include "datasourcefactory.h"
0012 #include "bitset.h"
0013 #include "settings.h"
0014 
0015 #include "core/filedeleter.h"
0016 #include "core/kget.h"
0017 #include "core/signature.h"
0018 #include "core/verifier.h"
0019 
0020 #include <cmath>
0021 
0022 #include <QDir>
0023 #include <QDomText>
0024 #include <QTimer>
0025 #include <QVarLengthArray>
0026 
0027 #include <KIO/CopyJob>
0028 #include <KIO/FileCopyJob>
0029 #include <KIO/FileJob>
0030 #include <KLocalizedString>
0031 #include <KMessageBox>
0032 #include <KMountPoint>
0033 
0034 #include "kget_debug.h"
0035 
0036 #include <qplatformdefs.h>
0037 
0038 const int SPEEDTIMER = 1000; // 1 second...
0039 
0040 DataSourceFactory::DataSourceFactory(QObject *parent, const QUrl &dest, KIO::filesize_t size, KIO::fileoffset_t segSize)
0041     : QObject(parent)
0042     , m_capabilities()
0043     , m_dest(dest)
0044     , m_size(size)
0045     , m_downloadedSize(0)
0046     , m_segSize(segSize)
0047     , m_speed(0)
0048     , m_percent(0)
0049     , m_tempOffset(0)
0050     , m_startedChunks(nullptr)
0051     , m_finishedChunks(nullptr)
0052     , m_putJob(nullptr)
0053     , m_doDownload(true)
0054     , m_open(false)
0055     , m_blocked(false)
0056     , m_startTried(false)
0057     , m_findFilesizeTried(false)
0058     , m_assignTried(false)
0059     , m_movingFile(false)
0060     , m_finished(false)
0061     , m_downloadInitialized(false)
0062     , m_sizeInitiallyDefined(m_size)
0063     , m_sizeFoundOnFinish(false)
0064     , m_maxMirrorsUsed(3)
0065     , m_speedTimer(nullptr)
0066     , m_status(Job::Stopped)
0067     , m_statusBeforeMove(m_status)
0068     , m_verifier(nullptr)
0069     , m_signature(nullptr)
0070 {
0071     qCDebug(KGET_DEBUG) << "Initialize DataSourceFactory: Dest: " + m_dest.toLocalFile() + "Size: " + QString::number(m_size)
0072             + "SegSize: " + QString::number(m_segSize);
0073 
0074     m_prevDownloadedSizes.append(0);
0075 }
0076 
0077 DataSourceFactory::~DataSourceFactory()
0078 {
0079     killPutJob();
0080     delete m_startedChunks;
0081     delete m_finishedChunks;
0082 }
0083 
0084 void DataSourceFactory::init()
0085 {
0086     if (!m_doDownload) {
0087         return;
0088     }
0089 
0090     if (!m_speedTimer) {
0091         m_speedTimer = new QTimer(this);
0092         m_speedTimer->setInterval(SPEEDTIMER);
0093         connect(m_speedTimer, &QTimer::timeout, this, &DataSourceFactory::speedChanged);
0094     }
0095 
0096     if (m_segSize && m_size) {
0097         const int hasRemainder = (m_size % m_segSize == 0) ? 0 : 1;
0098         const int bitSetSize = (m_size / m_segSize) + hasRemainder; // round up if needed
0099         if (!m_startedChunks && bitSetSize) {
0100             m_startedChunks = new BitSet(bitSetSize);
0101         }
0102         if (!m_finishedChunks && bitSetSize) {
0103             m_finishedChunks = new BitSet(bitSetSize);
0104         }
0105     }
0106 }
0107 
0108 void DataSourceFactory::deinit()
0109 {
0110     if (m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
0111         FileDeleter::deleteFile(m_dest);
0112     }
0113 }
0114 
0115 void DataSourceFactory::findFileSize()
0116 {
0117     qCDebug(KGET_DEBUG) << "Find the filesize" << this;
0118     if (!m_size && !m_dest.isEmpty() && !m_sources.isEmpty()) {
0119         foreach (TransferDataSource *source, m_sources) {
0120             if (source->capabilities() & Transfer::Cap_FindFilesize) {
0121                 connect(source, &TransferDataSource::foundFileSize, this, &DataSourceFactory::slotFoundFileSize);
0122                 connect(source, &TransferDataSource::finishedDownload, this, &DataSourceFactory::slotFinishedDownload);
0123 
0124                 m_speedTimer->start();
0125                 source->findFileSize(m_segSize);
0126                 changeStatus(Job::Running);
0127                 slotUpdateCapabilities();
0128                 return;
0129             }
0130         }
0131     }
0132 }
0133 
0134 void DataSourceFactory::slotFoundFileSize(TransferDataSource *source, KIO::filesize_t fileSize, const QPair<int, int> &segmentRange)
0135 {
0136     m_size = fileSize;
0137     qCDebug(KGET_DEBUG) << source << "found size" << m_size << "and is assigned segments" << segmentRange << this;
0138     Q_EMIT dataSourceFactoryChange(Transfer::Tc_TotalSize);
0139 
0140     init();
0141 
0142     if ((segmentRange.first != -1) && (segmentRange.second != -1)) {
0143         m_startedChunks->setRange(segmentRange.first, segmentRange.second, true);
0144     }
0145 
0146     if (m_startTried) {
0147         start();
0148     }
0149 }
0150 
0151 void DataSourceFactory::slotFinishedDownload(TransferDataSource *source, KIO::filesize_t size)
0152 {
0153     Q_UNUSED(source)
0154     Q_UNUSED(size)
0155 
0156     m_speedTimer->stop();
0157     m_finished = true;
0158 }
0159 
0160 bool DataSourceFactory::checkLocalFile()
0161 {
0162     QString dest_orig = m_dest.toLocalFile();
0163     QString _dest_part(dest_orig);
0164 
0165     QT_STATBUF buff_part;
0166     bool bPartExists = (QT_STAT(_dest_part.toUtf8().constData(), &buff_part) != -1);
0167     if (!bPartExists) {
0168         QString _dest = dest_orig;
0169         int fd = -1;
0170         mode_t initialMode = 0666;
0171 
0172         fd = QT_OPEN(_dest.toUtf8().constData(), O_CREAT | O_TRUNC | O_WRONLY, initialMode);
0173         if (fd < 0) {
0174             qCDebug(KGET_DEBUG) << " error";
0175             return false;
0176         } else {
0177             close(fd);
0178         }
0179     } // TODO: Port to use Qt functions maybe
0180 
0181     qCDebug(KGET_DEBUG) << "success";
0182     return true;
0183 }
0184 
0185 void DataSourceFactory::start()
0186 {
0187     qCDebug(KGET_DEBUG) << "Start DataSourceFactory";
0188     if (m_movingFile || (m_status == Job::Finished)) {
0189         return;
0190     }
0191     if (!m_doDownload) {
0192         m_startTried = true;
0193         return;
0194     }
0195 
0196     // the file already exists, even though DataSourceFactory has not been initialized remove it
0197     // to avoid problems like over methods not finished removing it because of a redownload
0198     if (!m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
0199         qCDebug(KGET_DEBUG) << "Removing existing file.";
0200         m_startTried = true;
0201         FileDeleter::deleteFile(m_dest, this, SLOT(slotRemovedFile()));
0202         return;
0203     }
0204 
0205     m_downloadInitialized = true;
0206 
0207     // create all dirs needed
0208     QDir dir;
0209     dir.mkpath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
0210     if (checkLocalFile()) {
0211         if (!m_putJob) {
0212             m_putJob = KIO::open(m_dest, QIODevice::WriteOnly | QIODevice::ReadOnly);
0213             connect(m_putJob, &KIO::FileJob::open, this, &DataSourceFactory::slotOpen);
0214             connect(m_putJob, &QObject::destroyed, this, &DataSourceFactory::slotPutJobDestroyed);
0215             m_startTried = true;
0216             return;
0217         }
0218     } else {
0219         // could not create file, maybe device not mounted so abort
0220         m_startTried = true;
0221         changeStatus(Job::Aborted);
0222         return;
0223     }
0224 
0225     init();
0226 
0227     if (!m_open) {
0228         m_startTried = true;
0229         return;
0230     }
0231 
0232     if (!m_size) {
0233         if (!m_findFilesizeTried && m_sources.count()) {
0234             m_findFilesizeTried = true;
0235             findFileSize();
0236         }
0237         m_startTried = true;
0238         return;
0239     }
0240 
0241     if (assignNeeded()) {
0242         if (m_sources.count()) {
0243             qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
0244             // simply assign a TransferDataSource
0245             assignSegments(*m_sources.begin());
0246         } else if (m_unusedUrls.count()) {
0247             qCDebug(KGET_DEBUG) << "Assigning an unused mirror";
0248             // takes the first unused mirror
0249             addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
0250         }
0251     }
0252 
0253     if (m_assignTried) {
0254         m_assignTried = false;
0255 
0256         foreach (TransferDataSource *source, m_sources) {
0257             assignSegments(source);
0258         }
0259     }
0260 
0261     if (m_open) {
0262         // check if the filesystem supports a file of m_size
0263         const static KIO::filesize_t maxFatSize = 4294967295;
0264         if (m_size > maxFatSize) {
0265             KMountPoint::Ptr mountPoint = KMountPoint::currentMountPoints().findByPath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
0266             if (mountPoint) {
0267                 if (mountPoint->mountType() == "vfat") { // TODO check what is reported on Windows for vfat
0268                     stop();
0269                     KMessageBox::error(nullptr, i18n("Filesize is larger than maximum file size supported by VFAT."), i18n("Error"));
0270                     return;
0271                 }
0272             }
0273         }
0274 
0275         QFile::resize(m_dest.toLocalFile(), m_size); // TODO should we keep that?
0276         m_speedTimer->start();
0277 
0278         foreach (TransferDataSource *source, m_sources) {
0279             source->start();
0280         }
0281 
0282         m_startTried = false;
0283         changeStatus(Job::Running);
0284     }
0285     slotUpdateCapabilities();
0286 }
0287 
0288 void DataSourceFactory::slotRemovedFile()
0289 {
0290     qCDebug(KGET_DEBUG) << "File has been removed" << this;
0291     if (m_startTried) {
0292         m_startTried = false;
0293         start();
0294     }
0295 }
0296 
0297 void DataSourceFactory::slotOpen(KIO::Job *job)
0298 {
0299     Q_UNUSED(job)
0300     qCDebug(KGET_DEBUG) << "File opened" << this;
0301 
0302     if (!m_speedTimer) {
0303         init();
0304     }
0305 
0306     connect(m_putJob, &KIO::FileJob::position, this, &DataSourceFactory::slotOffset);
0307     connect(m_putJob, &KIO::FileJob::written, this, &DataSourceFactory::slotDataWritten);
0308     m_open = true;
0309 
0310     if (m_startTried) {
0311         start();
0312     }
0313 }
0314 
0315 void DataSourceFactory::stop()
0316 {
0317     qCDebug(KGET_DEBUG) << "Stopping" << this;
0318     if (m_movingFile || (m_status == Job::Finished)) {
0319         return;
0320     }
0321 
0322     if (m_speedTimer) {
0323         m_speedTimer->stop();
0324     }
0325 
0326     foreach (TransferDataSource *source, m_sources) {
0327         source->stop();
0328     }
0329     m_startTried = false;
0330     m_findFilesizeTried = false;
0331     changeStatus(Job::Stopped);
0332 
0333     slotUpdateCapabilities();
0334 }
0335 
0336 void DataSourceFactory::setDoDownload(bool doDownload)
0337 {
0338     if (m_doDownload == doDownload) {
0339         return;
0340     }
0341 
0342     m_doDownload = doDownload;
0343     if (m_doDownload) {
0344         if (m_startTried) {
0345             start();
0346         }
0347     } else {
0348         if (m_status == Job::Running) {
0349             stop();
0350         }
0351     }
0352 }
0353 
0354 void DataSourceFactory::addMirror(const QUrl &url, int numParallelConnections)
0355 {
0356     addMirror(url, true, numParallelConnections, false);
0357 }
0358 
0359 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections)
0360 {
0361     addMirror(url, used, numParallelConnections, true);
0362 }
0363 
0364 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections, bool usedDefined)
0365 {
0366     if (!url.isValid() || url.isEmpty()) {
0367         qCDebug(KGET_DEBUG) << "Url is not usable: " << url.toString();
0368         return;
0369     }
0370     if (numParallelConnections <= 0) {
0371         numParallelConnections = 1;
0372     }
0373     if (!usedDefined) {
0374         used = true;
0375     }
0376 
0377     if (used) {
0378         // there is already a TransferDataSource with that url, reuse it and modify numParallelSegments
0379         if (m_sources.contains(url)) {
0380             TransferDataSource *source = m_sources[url];
0381             source->setParallelSegments(numParallelConnections);
0382             if (source->changeNeeded() > 0) {
0383                 assignSegments(source);
0384             } else {
0385                 for (int i = source->changeNeeded(); i < 0; ++i) {
0386                     const QPair<int, int> removed = source->removeConnection();
0387                     qCDebug(KGET_DEBUG) << "Remove connection with segments" << removed;
0388                     const int start = removed.first;
0389                     const int end = removed.second;
0390                     if ((start != -1) && (end != -1)) {
0391                         m_startedChunks->setRange(start, end, false);
0392                     }
0393                 }
0394             }
0395         } else {
0396             if (m_sources.count() < m_maxMirrorsUsed) {
0397                 TransferDataSource *source = KGet::createTransferDataSource(url, QDomElement(), this);
0398                 if (source) {
0399                     qCDebug(KGET_DEBUG) << "Successfully created a TransferDataSource for " << url.toString() << this;
0400 
0401                     // url might have been an unused Mirror, so remove it in any case
0402                     const int index = m_unusedUrls.indexOf(url);
0403                     if (index > -1) {
0404                         m_unusedUrls.removeAt(index);
0405                         m_unusedConnections.removeAt(index);
0406                     }
0407 
0408                     m_sources[url] = source;
0409                     m_sources[url]->setParallelSegments(numParallelConnections);
0410                     if (m_sizeInitiallyDefined) {
0411                         source->setSupposedSize(m_size);
0412                     }
0413 
0414                     connect(source, &TransferDataSource::capabilitiesChanged, this, &DataSourceFactory::slotUpdateCapabilities);
0415                     connect(source, &TransferDataSource::brokenSegments, this, &DataSourceFactory::brokenSegments);
0416                     connect(source, &TransferDataSource::broken, this, &DataSourceFactory::broken);
0417                     connect(source, &TransferDataSource::finishedSegment, this, &DataSourceFactory::finishedSegment);
0418                     connect(source,
0419                             qOverload<KIO::fileoffset_t, const QByteArray &, bool &>(&TransferDataSource::data),
0420                             this,
0421                             &DataSourceFactory::slotWriteData);
0422                     connect(source, &TransferDataSource::freeSegments, this, &DataSourceFactory::slotFreeSegments);
0423                     connect(source, &TransferDataSource::log, this, &DataSourceFactory::log);
0424                     connect(source, &TransferDataSource::urlChanged, this, &DataSourceFactory::slotUrlChanged);
0425 
0426                     slotUpdateCapabilities();
0427 
0428                     assignSegments(source);
0429 
0430                     // the job is already running, so also start the TransferDataSource
0431                     if (!m_assignTried && !m_startTried && m_putJob && m_open && (m_status == Job::Running)) {
0432                         source->start();
0433                     }
0434                 } else {
0435                     qCDebug(KGET_DEBUG) << "A TransferDataSource could not be created for " << url.toString();
0436                 }
0437             } else if (usedDefined) {
0438                 // increase the number of allowed mirrors as the user wants to use this one!
0439                 ++m_maxMirrorsUsed;
0440                 addMirror(url, used, numParallelConnections, usedDefined);
0441                 return;
0442             } else {
0443                 m_unusedUrls.append(url);
0444                 m_unusedConnections.append(numParallelConnections);
0445             }
0446         }
0447     } else {
0448         if (m_sources.contains(url)) {
0449             removeMirror(url);
0450         } else {
0451             m_unusedUrls.append(url);
0452             m_unusedConnections.append(numParallelConnections);
0453         }
0454     }
0455 }
0456 
0457 void DataSourceFactory::slotUrlChanged(const QUrl &old, const QUrl &newUrl)
0458 {
0459     TransferDataSource *ds = m_sources.take(old);
0460     m_sources[newUrl] = ds;
0461     Q_EMIT dataSourceFactoryChange(Transfer::Tc_Source | Transfer::Tc_FileName);
0462 }
0463 
0464 void DataSourceFactory::removeMirror(const QUrl &url)
0465 {
0466     qCDebug(KGET_DEBUG) << "Removing mirror: " << url;
0467     if (m_sources.contains(url)) {
0468         TransferDataSource *source = m_sources[url];
0469         source->stop();
0470         const QList<QPair<int, int>> assigned = source->assignedSegments();
0471         m_sources.remove(url);
0472         m_unusedUrls.append(url);
0473         m_unusedConnections.append(source->parallelSegments());
0474         delete source;
0475 
0476         for (int i = 0; i < assigned.count(); ++i) {
0477             const int start = assigned[i].first;
0478             const int end = assigned[i].second;
0479             if ((start != -1) && (end != -1)) {
0480                 m_startedChunks->setRange(start, end, false);
0481                 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
0482             }
0483         }
0484     }
0485 
0486     if ((m_status == Job::Running) && assignNeeded()) {
0487         // here we only handle the case when there are existing TransferDataSources,
0488         // the other case is triggered when stopping and then starting again
0489         if (m_sources.count()) {
0490             qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
0491             // simply assign a TransferDataSource
0492             assignSegments(*m_sources.begin());
0493         }
0494     }
0495 }
0496 
0497 void DataSourceFactory::setMirrors(const QHash<QUrl, QPair<bool, int>> &mirrors)
0498 {
0499     // first remove the not set DataSources
0500     QList<QUrl> oldUrls = m_sources.keys();
0501     QList<QUrl> newUrls = mirrors.keys();
0502 
0503     foreach (const QUrl &url, oldUrls) {
0504         if (!newUrls.contains(url)) {
0505             removeMirror(url);
0506         }
0507     }
0508 
0509     // remove all unused Mirrors and simply readd them below
0510     m_unusedUrls.clear();
0511     m_unusedConnections.clear();
0512 
0513     // second modify the existing DataSources and add the new ones
0514     QHash<QUrl, QPair<bool, int>>::const_iterator it;
0515     QHash<QUrl, QPair<bool, int>>::const_iterator itEnd = mirrors.constEnd();
0516     for (it = mirrors.constBegin(); it != itEnd; ++it) {
0517         addMirror(it.key(), it.value().first, it.value().second, true);
0518     }
0519 }
0520 
0521 QHash<QUrl, QPair<bool, int>> DataSourceFactory::mirrors() const
0522 {
0523     QHash<QUrl, QPair<bool, int>> mirrors;
0524 
0525     QHash<QUrl, TransferDataSource *>::const_iterator it;
0526     QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd();
0527     for (it = m_sources.constBegin(); it != itEnd; ++it) {
0528         mirrors[it.key()] = QPair<bool, int>(true, (*it)->parallelSegments());
0529     }
0530 
0531     for (int i = 0; i < m_unusedUrls.count(); ++i) {
0532         mirrors[m_unusedUrls[i]] = QPair<bool, int>(false, m_unusedConnections[i]);
0533     }
0534 
0535     return mirrors;
0536 }
0537 
0538 bool DataSourceFactory::assignNeeded() const
0539 {
0540     bool assignNeeded = true;
0541     QHash<QUrl, TransferDataSource *>::const_iterator it;
0542     QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd();
0543     for (it = m_sources.constBegin(); it != itEnd; ++it) {
0544         if ((*it)->currentSegments()) {
0545             // at least one TransferDataSource is still running, so no assign needed
0546             assignNeeded = false;
0547             break;
0548         }
0549     }
0550     return assignNeeded;
0551 }
0552 
0553 void DataSourceFactory::brokenSegments(TransferDataSource *source, const QPair<int, int> &segmentRange)
0554 {
0555     qCDebug(KGET_DEBUG) << "Segments" << segmentRange << "broken," << source;
0556     if (!source || !m_startedChunks || !m_finishedChunks || (segmentRange.first < 0) || (segmentRange.second < 0)
0557         || (static_cast<quint32>(segmentRange.second) > m_finishedChunks->getNumBits())) {
0558         return;
0559     }
0560 
0561     const int start = segmentRange.first;
0562     const int end = segmentRange.second;
0563     if ((start != -1) && (end != -1)) {
0564         m_startedChunks->setRange(start, end, false);
0565     }
0566 
0567     removeMirror(source->sourceUrl());
0568 }
0569 
0570 void DataSourceFactory::broken(TransferDataSource *source, TransferDataSource::Error error)
0571 {
0572     qCDebug(KGET_DEBUG) << source << "is broken with error" << error;
0573     const QString url = source->sourceUrl().toString();
0574 
0575     removeMirror(source->sourceUrl());
0576 
0577     if (error == TransferDataSource::WrongDownloadSize) {
0578         KMessageBox::error(nullptr,
0579                            i18nc("A mirror is removed when the file has the wrong download size", "%1 removed as it did report a wrong file size.", url),
0580                            i18n("Error"));
0581     }
0582 }
0583 
0584 void DataSourceFactory::slotFreeSegments(TransferDataSource *source, QPair<int, int> segmentRange)
0585 {
0586     qCDebug(KGET_DEBUG) << "Segments freed:" << source << segmentRange;
0587 
0588     const int start = segmentRange.first;
0589     const int end = segmentRange.second;
0590     if ((start != -1) && (end != -1)) {
0591         m_startedChunks->setRange(start, end, false);
0592         qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
0593     }
0594 
0595     assignSegments(source);
0596 }
0597 
0598 void DataSourceFactory::finishedSegment(TransferDataSource *source, int segmentNumber, bool connectionFinished)
0599 {
0600     if (!source || (segmentNumber < 0) || (static_cast<quint32>(segmentNumber) > m_finishedChunks->getNumBits())) {
0601         qCDebug(KGET_DEBUG) << "Incorrect data";
0602         return;
0603     }
0604 
0605     m_finishedChunks->set(segmentNumber, true);
0606 
0607     if (!connectionFinished) {
0608         qCDebug(KGET_DEBUG) << "Some segments still not finished";
0609         return;
0610     }
0611 
0612     m_finished = m_finishedChunks->allOn();
0613     if (m_finished) {
0614         qDebug() << "All segments have been downloaded.";
0615         return;
0616     }
0617 
0618     assignSegments(source);
0619 }
0620 
0621 void DataSourceFactory::assignSegments(TransferDataSource *source)
0622 {
0623     if (!m_startedChunks || !m_finishedChunks) {
0624         qCDebug(KGET_DEBUG) << "Assign tried";
0625         m_assignTried = true;
0626         return;
0627     }
0628     if (m_finishedChunks->allOn()) {
0629         qCDebug(KGET_DEBUG) << "All segments are finished already.";
0630         return;
0631     }
0632 
0633     // no more segments needed for that TransferDataSource
0634     if (source->changeNeeded() <= 0) {
0635         qCDebug(KGET_DEBUG) << "No change needed for" << source;
0636         return;
0637     }
0638 
0639     // find the segments that should be assigned to the transferDataSource
0640     int newStart = -1;
0641     int newEnd = -1;
0642 
0643     // a split needed
0644     if (m_startedChunks->allOn()) {
0645         int unfinished = 0;
0646         TransferDataSource *target = nullptr;
0647         foreach (TransferDataSource *source, m_sources) {
0648             int temp = source->countUnfinishedSegments();
0649             if (temp > unfinished) {
0650                 unfinished = temp;
0651                 target = source;
0652             }
0653         }
0654         if (!unfinished || !target) {
0655             return;
0656         }
0657 
0658         const QPair<int, int> splitResult = target->split();
0659         newStart = splitResult.first;
0660         newEnd = splitResult.second;
0661     } else {
0662         m_startedChunks->getContinuousRange(&newStart, &newEnd, false);
0663     }
0664 
0665     if ((newStart == -1) || (newEnd == -1)) {
0666         qCDebug(KGET_DEBUG) << "No segment can be assigned.";
0667         return;
0668     }
0669 
0670     const KIO::fileoffset_t rest = m_size % m_segSize;
0671 
0672     // the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
0673     const KIO::fileoffset_t lastSegSize = ((static_cast<uint>(newEnd + 1) == m_startedChunks->getNumBits() && rest) ? rest : m_segSize);
0674 
0675     qCDebug(KGET_DEBUG) << "Segments assigned:" << newStart << "-" << newEnd << "segment-size:" << m_segSize << "rest:" << rest;
0676     m_startedChunks->setRange(newStart, newEnd, true);
0677     source->addSegments(qMakePair(m_segSize, lastSegSize), qMakePair(newStart, newEnd));
0678 
0679     // there should still be segments added to this transfer
0680     if (source->changeNeeded() > 0) {
0681         assignSegments(source);
0682     }
0683 }
0684 
0685 // TODO implement checks if the correct offsets etc. are used + error recovering e.g. when something else
0686 // touches the file
0687 void DataSourceFactory::slotWriteData(KIO::fileoffset_t offset, const QByteArray &data, bool &worked)
0688 {
0689     worked = !m_blocked && !m_movingFile && m_open;
0690     if (m_blocked || m_movingFile || !m_open) {
0691         return;
0692     }
0693 
0694     m_blocked = true;
0695     m_tempOffset = offset;
0696     m_tempData = data;
0697     m_putJob->seek(offset);
0698 }
0699 
0700 void DataSourceFactory::slotOffset(KIO::Job *job, KIO::filesize_t offset)
0701 {
0702     Q_UNUSED(job)
0703     Q_UNUSED(offset)
0704 
0705     m_putJob->write(m_tempData);
0706 }
0707 
0708 void DataSourceFactory::slotDataWritten(KIO::Job *job, KIO::filesize_t written)
0709 {
0710     Q_UNUSED(job)
0711 
0712     auto tempSize = static_cast<KIO::filesize_t>(m_tempData.size());
0713     // the complete data has been written
0714     if (written == tempSize) // TODO if not same cache it temporarily!
0715     {
0716         m_downloadedSize += written;
0717         Q_EMIT dataSourceFactoryChange(Transfer::Tc_DownloadedSize);
0718         //             m_tempCache.clear();
0719     }
0720 
0721     if (m_finished) {
0722         m_speedTimer->stop();
0723         killPutJob();
0724         changeStatus(Job::Finished);
0725     }
0726     m_tempData.clear();
0727     m_blocked = false;
0728 }
0729 
0730 void DataSourceFactory::slotPercent(KJob *job, ulong p)
0731 {
0732     Q_UNUSED(job)
0733     m_percent = p;
0734     Q_EMIT dataSourceFactoryChange(Transfer::Tc_Percent);
0735 }
0736 
0737 void DataSourceFactory::speedChanged()
0738 {
0739     m_speed = (m_downloadedSize - m_prevDownloadedSizes.first()) / (SPEEDTIMER * m_prevDownloadedSizes.size() / 1000); // downloaded in 1 second
0740 
0741     m_prevDownloadedSizes.append(m_downloadedSize);
0742     if (m_prevDownloadedSizes.size() > 10)
0743         m_prevDownloadedSizes.removeFirst();
0744 
0745     ulong percent = (m_size ? (m_downloadedSize * 100 / m_size) : 0);
0746     const bool percentChanged = (percent != m_percent);
0747     m_percent = percent;
0748 
0749     Transfer::ChangesFlags change = (percentChanged ? (Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent) : Transfer::Tc_DownloadSpeed);
0750     Q_EMIT dataSourceFactoryChange(change);
0751 }
0752 
0753 void DataSourceFactory::killPutJob()
0754 {
0755     if (m_putJob) {
0756         qCDebug(KGET_DEBUG) << "Closing the file";
0757         m_open = false;
0758         m_putJob->close();
0759         m_putJob = nullptr;
0760     }
0761 }
0762 
0763 void DataSourceFactory::slotPutJobDestroyed(QObject *job)
0764 {
0765     Q_UNUSED(job)
0766 
0767     m_putJob = nullptr;
0768 }
0769 
0770 bool DataSourceFactory::setNewDestination(const QUrl &newDestination)
0771 {
0772     m_newDest = newDestination;
0773     if (m_newDest.isValid() && (m_newDest != m_dest)) {
0774         // No files created yet, simply change the urls
0775         if (!m_downloadInitialized) {
0776             m_dest = m_newDest;
0777             if (m_verifier) {
0778                 verifier()->setDestination(m_dest);
0779             }
0780             if (m_signature) {
0781                 signature()->setDestination(m_dest);
0782             }
0783 
0784             return true;
0785         } else if (QFile::exists(m_dest.toString())) {
0786             // create all dirs needed
0787             QDir dir;
0788             dir.mkpath(m_newDest.adjusted(QUrl::RemoveFilename).toString());
0789 
0790             m_statusBeforeMove = m_status;
0791             stop();
0792             changeStatus(Job::Moving);
0793             m_movingFile = true;
0794 
0795             // still a write in progress
0796             if (m_blocked) {
0797                 QTimer::singleShot(1000, this, &DataSourceFactory::startMove);
0798             } else {
0799                 startMove();
0800             }
0801             return true;
0802         }
0803     }
0804     return false;
0805 }
0806 
0807 void DataSourceFactory::startMove()
0808 {
0809     killPutJob();
0810 
0811     KIO::Job *move = KIO::file_move(m_dest, m_newDest, -1, KIO::HideProgressInfo);
0812     connect(move, &KJob::result, this, &DataSourceFactory::newDestResult);
0813     connect(move, &KJob::percentChanged, this, &DataSourceFactory::slotPercent);
0814 
0815     m_dest = m_newDest;
0816     verifier()->setDestination(m_dest);
0817     signature()->setDestination(m_dest);
0818 }
0819 
0820 void DataSourceFactory::newDestResult(KJob *job)
0821 {
0822     Q_UNUSED(job) // TODO handle errors etc.?
0823 
0824     m_movingFile = false;
0825     changeStatus(m_statusBeforeMove);
0826     if (m_status == Job::Running) {
0827         start();
0828     }
0829 }
0830 
0831 void DataSourceFactory::repair()
0832 {
0833     if (verifier()->status() != Verifier::NotVerified) {
0834         return;
0835     }
0836 
0837     m_finished = false;
0838 
0839     connect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>, KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>, KIO::filesize_t)));
0840 
0841     verifier()->brokenPieces();
0842 }
0843 
0844 void DataSourceFactory::slotRepair(const QList<KIO::fileoffset_t> &offsets, KIO::filesize_t length)
0845 {
0846     disconnect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>, KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>, KIO::filesize_t)));
0847 
0848     if (!m_startedChunks || !m_finishedChunks) {
0849         qCDebug(KGET_DEBUG) << "Redownload everything";
0850         m_downloadedSize = 0;
0851     } else {
0852         if (offsets.isEmpty()) {
0853             m_startedChunks->clear();
0854             m_finishedChunks->clear();
0855         }
0856         qCDebug(KGET_DEBUG) << "Redownload broken pieces";
0857         for (int i = 0; i < offsets.count(); ++i) {
0858             const int start = offsets[i] / m_segSize;
0859             const int end = std::ceil(length / static_cast<double>(m_segSize)) - 1 + start;
0860             m_startedChunks->setRange(start, end, false);
0861             m_finishedChunks->setRange(start, end, false);
0862         }
0863 
0864         m_downloadedSize = m_segSize * m_finishedChunks->numOnBits();
0865     }
0866     m_prevDownloadedSizes.clear();
0867     m_prevDownloadedSizes.append(m_downloadedSize);
0868 
0869     // remove all current mirrors and readd the first unused mirror
0870     const QList<QUrl> mirrors =
0871         m_sources.keys(); // FIXME only remove the mirrors of the broken segments?! --> for that m_assignedChunks would need to be saved was well
0872     foreach (const QUrl &mirror, mirrors) {
0873         removeMirror(mirror);
0874     }
0875     addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
0876 
0877     m_speed = 0;
0878 
0879     Transfer::ChangesFlags change = (Transfer::Tc_DownloadSpeed | Transfer::Tc_DownloadedSize);
0880     if (m_size) {
0881         change |= Transfer::Tc_Percent;
0882         m_percent = (m_downloadedSize * 100 / m_size);
0883     }
0884     Q_EMIT dataSourceFactoryChange(change);
0885     m_status = Job::Stopped;
0886 
0887     start();
0888 }
0889 
0890 void DataSourceFactory::load(const QDomElement *element)
0891 {
0892     if (!element) {
0893         return;
0894     }
0895 
0896     QDomElement e = element->firstChildElement("factory");
0897     if (e.isNull()) {
0898         e = element->firstChildElement("factories").firstChildElement("factory");
0899     }
0900 
0901     // only try to load values if they haven't been set
0902     if (m_dest.isEmpty()) {
0903         m_dest = QUrl(e.attribute("dest"));
0904     }
0905 
0906     verifier()->load(e);
0907     signature()->load(e);
0908 
0909     Transfer::ChangesFlags change = Transfer::Tc_None;
0910 
0911     if (!m_size) {
0912         m_size = e.attribute("size").toULongLong();
0913         change |= Transfer::Tc_TotalSize;
0914     }
0915     KIO::fileoffset_t tempSegSize = e.attribute("segmentSize").toLongLong();
0916     if (tempSegSize) {
0917         m_segSize = tempSegSize;
0918     }
0919     if (!m_downloadedSize) {
0920         m_downloadedSize = e.attribute("processedSize").toULongLong();
0921         change |= Transfer::Tc_DownloadedSize;
0922         if (m_size) {
0923             m_percent = (m_downloadedSize * 100 / m_size);
0924             change |= Transfer::Tc_Percent;
0925         }
0926     }
0927     if (e.hasAttribute("doDownload")) {
0928         m_doDownload = QVariant(e.attribute("doDownload")).toBool();
0929     }
0930     if (e.hasAttribute("downloadInitialized")) {
0931         m_downloadInitialized = QVariant(e.attribute("downloadInitialized")).toBool();
0932     }
0933     if (e.hasAttribute("maxMirrorsUsed")) {
0934         bool worked;
0935         m_maxMirrorsUsed = e.attribute("maxMirrorsUsed").toInt(&worked);
0936         m_maxMirrorsUsed = (worked ? m_maxMirrorsUsed : 3);
0937     }
0938     m_sizeInitiallyDefined = QVariant(e.attribute("sizeInitiallyDefined", "false")).toBool();
0939     m_sizeFoundOnFinish = QVariant(e.attribute("sizeFoundOnFinish", "false")).toBool();
0940 
0941     // load the finishedChunks
0942     const QDomElement chunks = e.firstChildElement("chunks");
0943     const QDomNodeList chunkList = chunks.elementsByTagName("chunk");
0944 
0945     const quint32 numBits = chunks.attribute("numBits").toInt();
0946     const int numBytes = chunks.attribute("numBytes").toInt();
0947     QVarLengthArray<quint8> data(numBytes);
0948 
0949     if (numBytes && (numBytes == chunkList.length())) {
0950         for (int i = 0; i < numBytes; ++i) {
0951             const quint8 value = chunkList.at(i).toElement().text().toInt();
0952             data[i] = value;
0953         }
0954 
0955         if (!m_finishedChunks) {
0956             m_finishedChunks = new BitSet(data.data(), numBits);
0957             qCDebug(KGET_DEBUG) << m_finishedChunks->numOnBits() << " bits on of " << numBits << " bits.";
0958         }
0959 
0960         // set the finished chunks to started
0961         if (!m_startedChunks) {
0962             m_startedChunks = new BitSet(data.data(), numBits);
0963         }
0964     }
0965     m_prevDownloadedSizes.clear();
0966     m_prevDownloadedSizes.append(m_downloadedSize);
0967 
0968     init();
0969 
0970     // add the used urls
0971     const QDomNodeList urls = e.firstChildElement("urls").elementsByTagName("url");
0972     for (int i = 0; i < urls.count(); ++i) {
0973         const QDomElement urlElement = urls.at(i).toElement();
0974         const QUrl url = QUrl(urlElement.text());
0975         const int connections = urlElement.attribute("numParallelSegments").toInt();
0976         addMirror(url, true, connections, true);
0977     }
0978 
0979     // add the unused urls
0980     const QDomNodeList unusedUrls = e.firstChildElement("unusedUrls").elementsByTagName("url");
0981     for (int i = 0; i < unusedUrls.count(); ++i) {
0982         const QDomElement urlElement = unusedUrls.at(i).toElement();
0983         const QUrl url = QUrl(urlElement.text());
0984         const int connections = urlElement.attribute("numParallelSegments").toInt();
0985         addMirror(url, false, connections, true);
0986     }
0987 
0988     // m_status = static_cast<Job::Status>(e.attribute("status").toInt());
0989 
0990     if (change != Transfer::Tc_None) {
0991         Q_EMIT dataSourceFactoryChange(change);
0992     }
0993 }
0994 
0995 void DataSourceFactory::changeStatus(Job::Status status)
0996 {
0997     Transfer::ChangesFlags change = Transfer::Tc_Status;
0998     m_status = status;
0999 
1000     switch (m_status) {
1001     case Job::Aborted:
1002     case Job::Moving:
1003     case Job::Stopped:
1004         m_speed = 0;
1005         change |= Transfer::Tc_DownloadSpeed;
1006         break;
1007     case Job::Running:
1008         break;
1009     case Job::Finished:
1010         m_speed = 0;
1011         m_percent = 100;
1012 
1013         if (m_size) {
1014             m_downloadedSize = m_size;
1015             change |= Transfer::Tc_DownloadedSize;
1016         } else if (m_downloadedSize) {
1017             m_sizeFoundOnFinish = true;
1018             m_size = m_downloadedSize;
1019             change |= Transfer::Tc_TotalSize;
1020         }
1021 
1022         change |= Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent;
1023 
1024         if (Settings::checksumAutomaticVerification() && verifier()->isVerifyable()) {
1025             verifier()->verify();
1026         }
1027         if (Settings::signatureAutomaticVerification() && signature()->isVerifyable()) {
1028             signature()->verify();
1029         }
1030 
1031         slotUpdateCapabilities();
1032         break;
1033     default:
1034         // TODO handle Delayed
1035         break;
1036     }
1037 
1038     Q_EMIT dataSourceFactoryChange(change);
1039 }
1040 
1041 void DataSourceFactory::save(const QDomElement &element)
1042 {
1043     if (element.isNull()) {
1044         return;
1045     }
1046 
1047     QDomElement e = element;
1048     QDomDocument doc(e.ownerDocument());
1049 
1050     QDomElement factories = e.firstChildElement("factories");
1051     if (factories.isNull()) {
1052         factories = doc.createElement("factories");
1053     }
1054 
1055     QDomElement factory = doc.createElement("factory");
1056     factory.setAttribute("dest", m_dest.url());
1057 
1058     if (!m_finishedChunks || m_sizeFoundOnFinish) {
1059         factory.setAttribute("processedSize", m_downloadedSize);
1060     }
1061     factory.setAttribute("size", m_size);
1062     factory.setAttribute("segmentSize", m_segSize);
1063     factory.setAttribute("status", m_status);
1064     factory.setAttribute("doDownload", m_doDownload);
1065     factory.setAttribute("downloadInitialized", m_downloadInitialized);
1066     factory.setAttribute("maxMirrorsUsed", m_maxMirrorsUsed);
1067     factory.setAttribute("sizeInitiallyDefined", m_sizeInitiallyDefined);
1068     factory.setAttribute("sizeFoundOnFinish", m_sizeFoundOnFinish);
1069 
1070     verifier()->save(factory);
1071     signature()->save(factory);
1072 
1073     // set the finished chunks, but only if chunks were actually used
1074     if (m_finishedChunks && !m_sizeFoundOnFinish) {
1075         const KIO::fileoffset_t rest = m_size % m_segSize;
1076         // the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
1077         const KIO::fileoffset_t lastSegSize = (rest ? rest : m_segSize);
1078 
1079         // not m_downloadedSize is stored, but the bytes that do not have to be redownloaded
1080         const bool lastOn = m_finishedChunks->get(m_finishedChunks->getNumBits() - 1);
1081         factory.setAttribute("processedSize", m_segSize * (m_finishedChunks->numOnBits() - lastOn) + lastOn * lastSegSize);
1082 
1083         QDomElement chunks = doc.createElement("chunks");
1084         chunks.setAttribute("numBits", m_finishedChunks->getNumBits());
1085         chunks.setAttribute("numBytes", m_finishedChunks->getNumBytes());
1086 
1087         const quint8 *data = m_finishedChunks->getData();
1088         for (quint32 i = 0; i < m_finishedChunks->getNumBytes(); ++i) {
1089             QDomElement chunk = doc.createElement("chunk");
1090             QDomText num = doc.createTextNode(QString::number(data[i]));
1091             chunk.setAttribute("number", i);
1092             chunk.appendChild(num);
1093             chunks.appendChild(chunk);
1094         }
1095         factory.appendChild(chunks);
1096     }
1097 
1098     // set the used urls
1099     QDomElement urls = doc.createElement("urls");
1100     QHash<QUrl, TransferDataSource *>::const_iterator it;
1101     QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd();
1102     for (it = m_sources.constBegin(); it != itEnd; ++it) {
1103         QDomElement url = doc.createElement("url");
1104         const QDomText text = doc.createTextNode(it.key().url());
1105         url.appendChild(text);
1106         url.setAttribute("numParallelSegments", (*it)->parallelSegments());
1107         urls.appendChild(url);
1108         factory.appendChild(urls);
1109     }
1110 
1111     // set the unused urls
1112     urls = doc.createElement("unusedUrls");
1113     for (int i = 0; i < m_unusedUrls.count(); ++i) {
1114         QDomElement url = doc.createElement("url");
1115         const QDomText text = doc.createTextNode(m_unusedUrls.at(i).url());
1116         url.appendChild(text);
1117         url.setAttribute("numParallelSegments", m_unusedConnections.at(i));
1118         urls.appendChild(url);
1119         factory.appendChild(urls);
1120     }
1121 
1122     factories.appendChild(factory);
1123     e.appendChild(factories);
1124 }
1125 
1126 bool DataSourceFactory::isValid() const
1127 {
1128     // FIXME
1129     return true;
1130     bool valid = (m_size && m_segSize && m_dest.isValid() && !m_sources.isEmpty());
1131 
1132     // if the download is finished the it is valid no matter what is set or not
1133     if (m_status == Job::Finished) {
1134         valid = true;
1135     }
1136     qDebug() << m_size << m_segSize << m_dest.isValid() << !m_sources.isEmpty();
1137     return valid;
1138 }
1139 
1140 Verifier *DataSourceFactory::verifier()
1141 {
1142     if (!m_verifier) {
1143         m_verifier = new Verifier(m_dest, this);
1144     }
1145 
1146     return m_verifier;
1147 }
1148 
1149 Signature *DataSourceFactory::signature()
1150 {
1151     if (!m_signature) {
1152         m_signature = new Signature(m_dest, this);
1153     }
1154 
1155     return m_signature;
1156 }
1157 
1158 void DataSourceFactory::slotUpdateCapabilities()
1159 {
1160     const Transfer::Capabilities oldCaps = capabilities();
1161     Transfer::Capabilities newCaps = {};
1162 
1163     if ((status() == Job::Finished) || (status() == Job::Stopped)) {
1164         newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1165     } else {
1166         foreach (TransferDataSource *source, m_sources) {
1167             if (!source->assignedSegments().isEmpty()) {
1168                 if (newCaps) {
1169                     newCaps &= source->capabilities();
1170                 } else {
1171                     newCaps = source->capabilities();
1172                 }
1173             }
1174         }
1175     }
1176 
1177     if (newCaps & Transfer::Cap_Resuming) {
1178         newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1179     }
1180 
1181     newCaps |= Transfer::Cap_MultipleMirrors;
1182 
1183     if (oldCaps != newCaps) {
1184         m_capabilities = newCaps;
1185         Q_EMIT capabilitiesChanged();
1186     }
1187 }
1188 
1189 #include "moc_datasourcefactory.cpp"