File indexing completed on 2025-02-09 05:51:38
0001 /* This file is part of the KDE project 0002 0003 Copyright (C) 2008 Lukas Appelhans <l.appelhans@gmx.de> 0004 Copyright (C) 2009 Matthias Fuchs <mat69@gmx.net> 0005 0006 This program is free software; you can redistribute it and/or 0007 modify it under the terms of the GNU General Public 0008 License as published by the Free Software Foundation; either 0009 version 2 of the License, or (at your option) any later version. 0010 */ 0011 #include "datasourcefactory.h" 0012 #include "bitset.h" 0013 #include "settings.h" 0014 0015 #include "core/filedeleter.h" 0016 #include "core/kget.h" 0017 #include "core/signature.h" 0018 #include "core/verifier.h" 0019 0020 #include <cmath> 0021 0022 #include <QDir> 0023 #include <QDomText> 0024 #include <QTimer> 0025 #include <QVarLengthArray> 0026 0027 #include <KIO/CopyJob> 0028 #include <KIO/FileCopyJob> 0029 #include <KIO/FileJob> 0030 #include <KLocalizedString> 0031 #include <KMessageBox> 0032 #include <KMountPoint> 0033 0034 #include "kget_debug.h" 0035 0036 #include <qplatformdefs.h> 0037 0038 const int SPEEDTIMER = 1000; // 1 second... 0039 0040 DataSourceFactory::DataSourceFactory(QObject *parent, const QUrl &dest, KIO::filesize_t size, KIO::fileoffset_t segSize) 0041 : QObject(parent) 0042 , m_capabilities() 0043 , m_dest(dest) 0044 , m_size(size) 0045 , m_downloadedSize(0) 0046 , m_segSize(segSize) 0047 , m_speed(0) 0048 , m_percent(0) 0049 , m_tempOffset(0) 0050 , m_startedChunks(nullptr) 0051 , m_finishedChunks(nullptr) 0052 , m_putJob(nullptr) 0053 , m_doDownload(true) 0054 , m_open(false) 0055 , m_blocked(false) 0056 , m_startTried(false) 0057 , m_findFilesizeTried(false) 0058 , m_assignTried(false) 0059 , m_movingFile(false) 0060 , m_finished(false) 0061 , m_downloadInitialized(false) 0062 , m_sizeInitiallyDefined(m_size) 0063 , m_sizeFoundOnFinish(false) 0064 , m_maxMirrorsUsed(3) 0065 , m_speedTimer(nullptr) 0066 , m_status(Job::Stopped) 0067 , m_statusBeforeMove(m_status) 0068 , m_verifier(nullptr) 0069 , m_signature(nullptr) 0070 { 0071 qCDebug(KGET_DEBUG) << "Initialize DataSourceFactory: Dest: " + m_dest.toLocalFile() + "Size: " + QString::number(m_size) 0072 + "SegSize: " + QString::number(m_segSize); 0073 0074 m_prevDownloadedSizes.append(0); 0075 } 0076 0077 DataSourceFactory::~DataSourceFactory() 0078 { 0079 killPutJob(); 0080 delete m_startedChunks; 0081 delete m_finishedChunks; 0082 } 0083 0084 void DataSourceFactory::init() 0085 { 0086 if (!m_doDownload) { 0087 return; 0088 } 0089 0090 if (!m_speedTimer) { 0091 m_speedTimer = new QTimer(this); 0092 m_speedTimer->setInterval(SPEEDTIMER); 0093 connect(m_speedTimer, &QTimer::timeout, this, &DataSourceFactory::speedChanged); 0094 } 0095 0096 if (m_segSize && m_size) { 0097 const int hasRemainder = (m_size % m_segSize == 0) ? 0 : 1; 0098 const int bitSetSize = (m_size / m_segSize) + hasRemainder; // round up if needed 0099 if (!m_startedChunks && bitSetSize) { 0100 m_startedChunks = new BitSet(bitSetSize); 0101 } 0102 if (!m_finishedChunks && bitSetSize) { 0103 m_finishedChunks = new BitSet(bitSetSize); 0104 } 0105 } 0106 } 0107 0108 void DataSourceFactory::deinit() 0109 { 0110 if (m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) { 0111 FileDeleter::deleteFile(m_dest); 0112 } 0113 } 0114 0115 void DataSourceFactory::findFileSize() 0116 { 0117 qCDebug(KGET_DEBUG) << "Find the filesize" << this; 0118 if (!m_size && !m_dest.isEmpty() && !m_sources.isEmpty()) { 0119 foreach (TransferDataSource *source, m_sources) { 0120 if (source->capabilities() & Transfer::Cap_FindFilesize) { 0121 connect(source, &TransferDataSource::foundFileSize, this, &DataSourceFactory::slotFoundFileSize); 0122 connect(source, &TransferDataSource::finishedDownload, this, &DataSourceFactory::slotFinishedDownload); 0123 0124 m_speedTimer->start(); 0125 source->findFileSize(m_segSize); 0126 changeStatus(Job::Running); 0127 slotUpdateCapabilities(); 0128 return; 0129 } 0130 } 0131 } 0132 } 0133 0134 void DataSourceFactory::slotFoundFileSize(TransferDataSource *source, KIO::filesize_t fileSize, const QPair<int, int> &segmentRange) 0135 { 0136 m_size = fileSize; 0137 qCDebug(KGET_DEBUG) << source << "found size" << m_size << "and is assigned segments" << segmentRange << this; 0138 Q_EMIT dataSourceFactoryChange(Transfer::Tc_TotalSize); 0139 0140 init(); 0141 0142 if ((segmentRange.first != -1) && (segmentRange.second != -1)) { 0143 m_startedChunks->setRange(segmentRange.first, segmentRange.second, true); 0144 } 0145 0146 if (m_startTried) { 0147 start(); 0148 } 0149 } 0150 0151 void DataSourceFactory::slotFinishedDownload(TransferDataSource *source, KIO::filesize_t size) 0152 { 0153 Q_UNUSED(source) 0154 Q_UNUSED(size) 0155 0156 m_speedTimer->stop(); 0157 m_finished = true; 0158 } 0159 0160 bool DataSourceFactory::checkLocalFile() 0161 { 0162 QString dest_orig = m_dest.toLocalFile(); 0163 QString _dest_part(dest_orig); 0164 0165 QT_STATBUF buff_part; 0166 bool bPartExists = (QT_STAT(_dest_part.toUtf8().constData(), &buff_part) != -1); 0167 if (!bPartExists) { 0168 QString _dest = dest_orig; 0169 int fd = -1; 0170 mode_t initialMode = 0666; 0171 0172 fd = QT_OPEN(_dest.toUtf8().constData(), O_CREAT | O_TRUNC | O_WRONLY, initialMode); 0173 if (fd < 0) { 0174 qCDebug(KGET_DEBUG) << " error"; 0175 return false; 0176 } else { 0177 close(fd); 0178 } 0179 } // TODO: Port to use Qt functions maybe 0180 0181 qCDebug(KGET_DEBUG) << "success"; 0182 return true; 0183 } 0184 0185 void DataSourceFactory::start() 0186 { 0187 qCDebug(KGET_DEBUG) << "Start DataSourceFactory"; 0188 if (m_movingFile || (m_status == Job::Finished)) { 0189 return; 0190 } 0191 if (!m_doDownload) { 0192 m_startTried = true; 0193 return; 0194 } 0195 0196 // the file already exists, even though DataSourceFactory has not been initialized remove it 0197 // to avoid problems like over methods not finished removing it because of a redownload 0198 if (!m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) { 0199 qCDebug(KGET_DEBUG) << "Removing existing file."; 0200 m_startTried = true; 0201 FileDeleter::deleteFile(m_dest, this, SLOT(slotRemovedFile())); 0202 return; 0203 } 0204 0205 m_downloadInitialized = true; 0206 0207 // create all dirs needed 0208 QDir dir; 0209 dir.mkpath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile()); 0210 if (checkLocalFile()) { 0211 if (!m_putJob) { 0212 m_putJob = KIO::open(m_dest, QIODevice::WriteOnly | QIODevice::ReadOnly); 0213 connect(m_putJob, &KIO::FileJob::open, this, &DataSourceFactory::slotOpen); 0214 connect(m_putJob, &QObject::destroyed, this, &DataSourceFactory::slotPutJobDestroyed); 0215 m_startTried = true; 0216 return; 0217 } 0218 } else { 0219 // could not create file, maybe device not mounted so abort 0220 m_startTried = true; 0221 changeStatus(Job::Aborted); 0222 return; 0223 } 0224 0225 init(); 0226 0227 if (!m_open) { 0228 m_startTried = true; 0229 return; 0230 } 0231 0232 if (!m_size) { 0233 if (!m_findFilesizeTried && m_sources.count()) { 0234 m_findFilesizeTried = true; 0235 findFileSize(); 0236 } 0237 m_startTried = true; 0238 return; 0239 } 0240 0241 if (assignNeeded()) { 0242 if (m_sources.count()) { 0243 qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource."; 0244 // simply assign a TransferDataSource 0245 assignSegments(*m_sources.begin()); 0246 } else if (m_unusedUrls.count()) { 0247 qCDebug(KGET_DEBUG) << "Assigning an unused mirror"; 0248 // takes the first unused mirror 0249 addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst()); 0250 } 0251 } 0252 0253 if (m_assignTried) { 0254 m_assignTried = false; 0255 0256 foreach (TransferDataSource *source, m_sources) { 0257 assignSegments(source); 0258 } 0259 } 0260 0261 if (m_open) { 0262 // check if the filesystem supports a file of m_size 0263 const static KIO::filesize_t maxFatSize = 4294967295; 0264 if (m_size > maxFatSize) { 0265 KMountPoint::Ptr mountPoint = KMountPoint::currentMountPoints().findByPath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile()); 0266 if (mountPoint) { 0267 if (mountPoint->mountType() == "vfat") { // TODO check what is reported on Windows for vfat 0268 stop(); 0269 KMessageBox::error(nullptr, i18n("Filesize is larger than maximum file size supported by VFAT."), i18n("Error")); 0270 return; 0271 } 0272 } 0273 } 0274 0275 QFile::resize(m_dest.toLocalFile(), m_size); // TODO should we keep that? 0276 m_speedTimer->start(); 0277 0278 foreach (TransferDataSource *source, m_sources) { 0279 source->start(); 0280 } 0281 0282 m_startTried = false; 0283 changeStatus(Job::Running); 0284 } 0285 slotUpdateCapabilities(); 0286 } 0287 0288 void DataSourceFactory::slotRemovedFile() 0289 { 0290 qCDebug(KGET_DEBUG) << "File has been removed" << this; 0291 if (m_startTried) { 0292 m_startTried = false; 0293 start(); 0294 } 0295 } 0296 0297 void DataSourceFactory::slotOpen(KIO::Job *job) 0298 { 0299 Q_UNUSED(job) 0300 qCDebug(KGET_DEBUG) << "File opened" << this; 0301 0302 if (!m_speedTimer) { 0303 init(); 0304 } 0305 0306 connect(m_putJob, &KIO::FileJob::position, this, &DataSourceFactory::slotOffset); 0307 connect(m_putJob, &KIO::FileJob::written, this, &DataSourceFactory::slotDataWritten); 0308 m_open = true; 0309 0310 if (m_startTried) { 0311 start(); 0312 } 0313 } 0314 0315 void DataSourceFactory::stop() 0316 { 0317 qCDebug(KGET_DEBUG) << "Stopping" << this; 0318 if (m_movingFile || (m_status == Job::Finished)) { 0319 return; 0320 } 0321 0322 if (m_speedTimer) { 0323 m_speedTimer->stop(); 0324 } 0325 0326 foreach (TransferDataSource *source, m_sources) { 0327 source->stop(); 0328 } 0329 m_startTried = false; 0330 m_findFilesizeTried = false; 0331 changeStatus(Job::Stopped); 0332 0333 slotUpdateCapabilities(); 0334 } 0335 0336 void DataSourceFactory::setDoDownload(bool doDownload) 0337 { 0338 if (m_doDownload == doDownload) { 0339 return; 0340 } 0341 0342 m_doDownload = doDownload; 0343 if (m_doDownload) { 0344 if (m_startTried) { 0345 start(); 0346 } 0347 } else { 0348 if (m_status == Job::Running) { 0349 stop(); 0350 } 0351 } 0352 } 0353 0354 void DataSourceFactory::addMirror(const QUrl &url, int numParallelConnections) 0355 { 0356 addMirror(url, true, numParallelConnections, false); 0357 } 0358 0359 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections) 0360 { 0361 addMirror(url, used, numParallelConnections, true); 0362 } 0363 0364 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections, bool usedDefined) 0365 { 0366 if (!url.isValid() || url.isEmpty()) { 0367 qCDebug(KGET_DEBUG) << "Url is not usable: " << url.toString(); 0368 return; 0369 } 0370 if (numParallelConnections <= 0) { 0371 numParallelConnections = 1; 0372 } 0373 if (!usedDefined) { 0374 used = true; 0375 } 0376 0377 if (used) { 0378 // there is already a TransferDataSource with that url, reuse it and modify numParallelSegments 0379 if (m_sources.contains(url)) { 0380 TransferDataSource *source = m_sources[url]; 0381 source->setParallelSegments(numParallelConnections); 0382 if (source->changeNeeded() > 0) { 0383 assignSegments(source); 0384 } else { 0385 for (int i = source->changeNeeded(); i < 0; ++i) { 0386 const QPair<int, int> removed = source->removeConnection(); 0387 qCDebug(KGET_DEBUG) << "Remove connection with segments" << removed; 0388 const int start = removed.first; 0389 const int end = removed.second; 0390 if ((start != -1) && (end != -1)) { 0391 m_startedChunks->setRange(start, end, false); 0392 } 0393 } 0394 } 0395 } else { 0396 if (m_sources.count() < m_maxMirrorsUsed) { 0397 TransferDataSource *source = KGet::createTransferDataSource(url, QDomElement(), this); 0398 if (source) { 0399 qCDebug(KGET_DEBUG) << "Successfully created a TransferDataSource for " << url.toString() << this; 0400 0401 // url might have been an unused Mirror, so remove it in any case 0402 const int index = m_unusedUrls.indexOf(url); 0403 if (index > -1) { 0404 m_unusedUrls.removeAt(index); 0405 m_unusedConnections.removeAt(index); 0406 } 0407 0408 m_sources[url] = source; 0409 m_sources[url]->setParallelSegments(numParallelConnections); 0410 if (m_sizeInitiallyDefined) { 0411 source->setSupposedSize(m_size); 0412 } 0413 0414 connect(source, &TransferDataSource::capabilitiesChanged, this, &DataSourceFactory::slotUpdateCapabilities); 0415 connect(source, &TransferDataSource::brokenSegments, this, &DataSourceFactory::brokenSegments); 0416 connect(source, &TransferDataSource::broken, this, &DataSourceFactory::broken); 0417 connect(source, &TransferDataSource::finishedSegment, this, &DataSourceFactory::finishedSegment); 0418 connect(source, 0419 qOverload<KIO::fileoffset_t, const QByteArray &, bool &>(&TransferDataSource::data), 0420 this, 0421 &DataSourceFactory::slotWriteData); 0422 connect(source, &TransferDataSource::freeSegments, this, &DataSourceFactory::slotFreeSegments); 0423 connect(source, &TransferDataSource::log, this, &DataSourceFactory::log); 0424 connect(source, &TransferDataSource::urlChanged, this, &DataSourceFactory::slotUrlChanged); 0425 0426 slotUpdateCapabilities(); 0427 0428 assignSegments(source); 0429 0430 // the job is already running, so also start the TransferDataSource 0431 if (!m_assignTried && !m_startTried && m_putJob && m_open && (m_status == Job::Running)) { 0432 source->start(); 0433 } 0434 } else { 0435 qCDebug(KGET_DEBUG) << "A TransferDataSource could not be created for " << url.toString(); 0436 } 0437 } else if (usedDefined) { 0438 // increase the number of allowed mirrors as the user wants to use this one! 0439 ++m_maxMirrorsUsed; 0440 addMirror(url, used, numParallelConnections, usedDefined); 0441 return; 0442 } else { 0443 m_unusedUrls.append(url); 0444 m_unusedConnections.append(numParallelConnections); 0445 } 0446 } 0447 } else { 0448 if (m_sources.contains(url)) { 0449 removeMirror(url); 0450 } else { 0451 m_unusedUrls.append(url); 0452 m_unusedConnections.append(numParallelConnections); 0453 } 0454 } 0455 } 0456 0457 void DataSourceFactory::slotUrlChanged(const QUrl &old, const QUrl &newUrl) 0458 { 0459 TransferDataSource *ds = m_sources.take(old); 0460 m_sources[newUrl] = ds; 0461 Q_EMIT dataSourceFactoryChange(Transfer::Tc_Source | Transfer::Tc_FileName); 0462 } 0463 0464 void DataSourceFactory::removeMirror(const QUrl &url) 0465 { 0466 qCDebug(KGET_DEBUG) << "Removing mirror: " << url; 0467 if (m_sources.contains(url)) { 0468 TransferDataSource *source = m_sources[url]; 0469 source->stop(); 0470 const QList<QPair<int, int>> assigned = source->assignedSegments(); 0471 m_sources.remove(url); 0472 m_unusedUrls.append(url); 0473 m_unusedConnections.append(source->parallelSegments()); 0474 delete source; 0475 0476 for (int i = 0; i < assigned.count(); ++i) { 0477 const int start = assigned[i].first; 0478 const int end = assigned[i].second; 0479 if ((start != -1) && (end != -1)) { 0480 m_startedChunks->setRange(start, end, false); 0481 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore."; 0482 } 0483 } 0484 } 0485 0486 if ((m_status == Job::Running) && assignNeeded()) { 0487 // here we only handle the case when there are existing TransferDataSources, 0488 // the other case is triggered when stopping and then starting again 0489 if (m_sources.count()) { 0490 qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource."; 0491 // simply assign a TransferDataSource 0492 assignSegments(*m_sources.begin()); 0493 } 0494 } 0495 } 0496 0497 void DataSourceFactory::setMirrors(const QHash<QUrl, QPair<bool, int>> &mirrors) 0498 { 0499 // first remove the not set DataSources 0500 QList<QUrl> oldUrls = m_sources.keys(); 0501 QList<QUrl> newUrls = mirrors.keys(); 0502 0503 foreach (const QUrl &url, oldUrls) { 0504 if (!newUrls.contains(url)) { 0505 removeMirror(url); 0506 } 0507 } 0508 0509 // remove all unused Mirrors and simply readd them below 0510 m_unusedUrls.clear(); 0511 m_unusedConnections.clear(); 0512 0513 // second modify the existing DataSources and add the new ones 0514 QHash<QUrl, QPair<bool, int>>::const_iterator it; 0515 QHash<QUrl, QPair<bool, int>>::const_iterator itEnd = mirrors.constEnd(); 0516 for (it = mirrors.constBegin(); it != itEnd; ++it) { 0517 addMirror(it.key(), it.value().first, it.value().second, true); 0518 } 0519 } 0520 0521 QHash<QUrl, QPair<bool, int>> DataSourceFactory::mirrors() const 0522 { 0523 QHash<QUrl, QPair<bool, int>> mirrors; 0524 0525 QHash<QUrl, TransferDataSource *>::const_iterator it; 0526 QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd(); 0527 for (it = m_sources.constBegin(); it != itEnd; ++it) { 0528 mirrors[it.key()] = QPair<bool, int>(true, (*it)->parallelSegments()); 0529 } 0530 0531 for (int i = 0; i < m_unusedUrls.count(); ++i) { 0532 mirrors[m_unusedUrls[i]] = QPair<bool, int>(false, m_unusedConnections[i]); 0533 } 0534 0535 return mirrors; 0536 } 0537 0538 bool DataSourceFactory::assignNeeded() const 0539 { 0540 bool assignNeeded = true; 0541 QHash<QUrl, TransferDataSource *>::const_iterator it; 0542 QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd(); 0543 for (it = m_sources.constBegin(); it != itEnd; ++it) { 0544 if ((*it)->currentSegments()) { 0545 // at least one TransferDataSource is still running, so no assign needed 0546 assignNeeded = false; 0547 break; 0548 } 0549 } 0550 return assignNeeded; 0551 } 0552 0553 void DataSourceFactory::brokenSegments(TransferDataSource *source, const QPair<int, int> &segmentRange) 0554 { 0555 qCDebug(KGET_DEBUG) << "Segments" << segmentRange << "broken," << source; 0556 if (!source || !m_startedChunks || !m_finishedChunks || (segmentRange.first < 0) || (segmentRange.second < 0) 0557 || (static_cast<quint32>(segmentRange.second) > m_finishedChunks->getNumBits())) { 0558 return; 0559 } 0560 0561 const int start = segmentRange.first; 0562 const int end = segmentRange.second; 0563 if ((start != -1) && (end != -1)) { 0564 m_startedChunks->setRange(start, end, false); 0565 } 0566 0567 removeMirror(source->sourceUrl()); 0568 } 0569 0570 void DataSourceFactory::broken(TransferDataSource *source, TransferDataSource::Error error) 0571 { 0572 qCDebug(KGET_DEBUG) << source << "is broken with error" << error; 0573 const QString url = source->sourceUrl().toString(); 0574 0575 removeMirror(source->sourceUrl()); 0576 0577 if (error == TransferDataSource::WrongDownloadSize) { 0578 KMessageBox::error(nullptr, 0579 i18nc("A mirror is removed when the file has the wrong download size", "%1 removed as it did report a wrong file size.", url), 0580 i18n("Error")); 0581 } 0582 } 0583 0584 void DataSourceFactory::slotFreeSegments(TransferDataSource *source, QPair<int, int> segmentRange) 0585 { 0586 qCDebug(KGET_DEBUG) << "Segments freed:" << source << segmentRange; 0587 0588 const int start = segmentRange.first; 0589 const int end = segmentRange.second; 0590 if ((start != -1) && (end != -1)) { 0591 m_startedChunks->setRange(start, end, false); 0592 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore."; 0593 } 0594 0595 assignSegments(source); 0596 } 0597 0598 void DataSourceFactory::finishedSegment(TransferDataSource *source, int segmentNumber, bool connectionFinished) 0599 { 0600 if (!source || (segmentNumber < 0) || (static_cast<quint32>(segmentNumber) > m_finishedChunks->getNumBits())) { 0601 qCDebug(KGET_DEBUG) << "Incorrect data"; 0602 return; 0603 } 0604 0605 m_finishedChunks->set(segmentNumber, true); 0606 0607 if (!connectionFinished) { 0608 qCDebug(KGET_DEBUG) << "Some segments still not finished"; 0609 return; 0610 } 0611 0612 m_finished = m_finishedChunks->allOn(); 0613 if (m_finished) { 0614 qDebug() << "All segments have been downloaded."; 0615 return; 0616 } 0617 0618 assignSegments(source); 0619 } 0620 0621 void DataSourceFactory::assignSegments(TransferDataSource *source) 0622 { 0623 if (!m_startedChunks || !m_finishedChunks) { 0624 qCDebug(KGET_DEBUG) << "Assign tried"; 0625 m_assignTried = true; 0626 return; 0627 } 0628 if (m_finishedChunks->allOn()) { 0629 qCDebug(KGET_DEBUG) << "All segments are finished already."; 0630 return; 0631 } 0632 0633 // no more segments needed for that TransferDataSource 0634 if (source->changeNeeded() <= 0) { 0635 qCDebug(KGET_DEBUG) << "No change needed for" << source; 0636 return; 0637 } 0638 0639 // find the segments that should be assigned to the transferDataSource 0640 int newStart = -1; 0641 int newEnd = -1; 0642 0643 // a split needed 0644 if (m_startedChunks->allOn()) { 0645 int unfinished = 0; 0646 TransferDataSource *target = nullptr; 0647 foreach (TransferDataSource *source, m_sources) { 0648 int temp = source->countUnfinishedSegments(); 0649 if (temp > unfinished) { 0650 unfinished = temp; 0651 target = source; 0652 } 0653 } 0654 if (!unfinished || !target) { 0655 return; 0656 } 0657 0658 const QPair<int, int> splitResult = target->split(); 0659 newStart = splitResult.first; 0660 newEnd = splitResult.second; 0661 } else { 0662 m_startedChunks->getContinuousRange(&newStart, &newEnd, false); 0663 } 0664 0665 if ((newStart == -1) || (newEnd == -1)) { 0666 qCDebug(KGET_DEBUG) << "No segment can be assigned."; 0667 return; 0668 } 0669 0670 const KIO::fileoffset_t rest = m_size % m_segSize; 0671 0672 // the lastSegsize is rest, but only if there is a rest and it is the last segment of the download 0673 const KIO::fileoffset_t lastSegSize = ((static_cast<uint>(newEnd + 1) == m_startedChunks->getNumBits() && rest) ? rest : m_segSize); 0674 0675 qCDebug(KGET_DEBUG) << "Segments assigned:" << newStart << "-" << newEnd << "segment-size:" << m_segSize << "rest:" << rest; 0676 m_startedChunks->setRange(newStart, newEnd, true); 0677 source->addSegments(qMakePair(m_segSize, lastSegSize), qMakePair(newStart, newEnd)); 0678 0679 // there should still be segments added to this transfer 0680 if (source->changeNeeded() > 0) { 0681 assignSegments(source); 0682 } 0683 } 0684 0685 // TODO implement checks if the correct offsets etc. are used + error recovering e.g. when something else 0686 // touches the file 0687 void DataSourceFactory::slotWriteData(KIO::fileoffset_t offset, const QByteArray &data, bool &worked) 0688 { 0689 worked = !m_blocked && !m_movingFile && m_open; 0690 if (m_blocked || m_movingFile || !m_open) { 0691 return; 0692 } 0693 0694 m_blocked = true; 0695 m_tempOffset = offset; 0696 m_tempData = data; 0697 m_putJob->seek(offset); 0698 } 0699 0700 void DataSourceFactory::slotOffset(KIO::Job *job, KIO::filesize_t offset) 0701 { 0702 Q_UNUSED(job) 0703 Q_UNUSED(offset) 0704 0705 m_putJob->write(m_tempData); 0706 } 0707 0708 void DataSourceFactory::slotDataWritten(KIO::Job *job, KIO::filesize_t written) 0709 { 0710 Q_UNUSED(job) 0711 0712 auto tempSize = static_cast<KIO::filesize_t>(m_tempData.size()); 0713 // the complete data has been written 0714 if (written == tempSize) // TODO if not same cache it temporarily! 0715 { 0716 m_downloadedSize += written; 0717 Q_EMIT dataSourceFactoryChange(Transfer::Tc_DownloadedSize); 0718 // m_tempCache.clear(); 0719 } 0720 0721 if (m_finished) { 0722 m_speedTimer->stop(); 0723 killPutJob(); 0724 changeStatus(Job::Finished); 0725 } 0726 m_tempData.clear(); 0727 m_blocked = false; 0728 } 0729 0730 void DataSourceFactory::slotPercent(KJob *job, ulong p) 0731 { 0732 Q_UNUSED(job) 0733 m_percent = p; 0734 Q_EMIT dataSourceFactoryChange(Transfer::Tc_Percent); 0735 } 0736 0737 void DataSourceFactory::speedChanged() 0738 { 0739 m_speed = (m_downloadedSize - m_prevDownloadedSizes.first()) / (SPEEDTIMER * m_prevDownloadedSizes.size() / 1000); // downloaded in 1 second 0740 0741 m_prevDownloadedSizes.append(m_downloadedSize); 0742 if (m_prevDownloadedSizes.size() > 10) 0743 m_prevDownloadedSizes.removeFirst(); 0744 0745 ulong percent = (m_size ? (m_downloadedSize * 100 / m_size) : 0); 0746 const bool percentChanged = (percent != m_percent); 0747 m_percent = percent; 0748 0749 Transfer::ChangesFlags change = (percentChanged ? (Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent) : Transfer::Tc_DownloadSpeed); 0750 Q_EMIT dataSourceFactoryChange(change); 0751 } 0752 0753 void DataSourceFactory::killPutJob() 0754 { 0755 if (m_putJob) { 0756 qCDebug(KGET_DEBUG) << "Closing the file"; 0757 m_open = false; 0758 m_putJob->close(); 0759 m_putJob = nullptr; 0760 } 0761 } 0762 0763 void DataSourceFactory::slotPutJobDestroyed(QObject *job) 0764 { 0765 Q_UNUSED(job) 0766 0767 m_putJob = nullptr; 0768 } 0769 0770 bool DataSourceFactory::setNewDestination(const QUrl &newDestination) 0771 { 0772 m_newDest = newDestination; 0773 if (m_newDest.isValid() && (m_newDest != m_dest)) { 0774 // No files created yet, simply change the urls 0775 if (!m_downloadInitialized) { 0776 m_dest = m_newDest; 0777 if (m_verifier) { 0778 verifier()->setDestination(m_dest); 0779 } 0780 if (m_signature) { 0781 signature()->setDestination(m_dest); 0782 } 0783 0784 return true; 0785 } else if (QFile::exists(m_dest.toString())) { 0786 // create all dirs needed 0787 QDir dir; 0788 dir.mkpath(m_newDest.adjusted(QUrl::RemoveFilename).toString()); 0789 0790 m_statusBeforeMove = m_status; 0791 stop(); 0792 changeStatus(Job::Moving); 0793 m_movingFile = true; 0794 0795 // still a write in progress 0796 if (m_blocked) { 0797 QTimer::singleShot(1000, this, &DataSourceFactory::startMove); 0798 } else { 0799 startMove(); 0800 } 0801 return true; 0802 } 0803 } 0804 return false; 0805 } 0806 0807 void DataSourceFactory::startMove() 0808 { 0809 killPutJob(); 0810 0811 KIO::Job *move = KIO::file_move(m_dest, m_newDest, -1, KIO::HideProgressInfo); 0812 connect(move, &KJob::result, this, &DataSourceFactory::newDestResult); 0813 connect(move, &KJob::percentChanged, this, &DataSourceFactory::slotPercent); 0814 0815 m_dest = m_newDest; 0816 verifier()->setDestination(m_dest); 0817 signature()->setDestination(m_dest); 0818 } 0819 0820 void DataSourceFactory::newDestResult(KJob *job) 0821 { 0822 Q_UNUSED(job) // TODO handle errors etc.? 0823 0824 m_movingFile = false; 0825 changeStatus(m_statusBeforeMove); 0826 if (m_status == Job::Running) { 0827 start(); 0828 } 0829 } 0830 0831 void DataSourceFactory::repair() 0832 { 0833 if (verifier()->status() != Verifier::NotVerified) { 0834 return; 0835 } 0836 0837 m_finished = false; 0838 0839 connect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>, KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>, KIO::filesize_t))); 0840 0841 verifier()->brokenPieces(); 0842 } 0843 0844 void DataSourceFactory::slotRepair(const QList<KIO::fileoffset_t> &offsets, KIO::filesize_t length) 0845 { 0846 disconnect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>, KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>, KIO::filesize_t))); 0847 0848 if (!m_startedChunks || !m_finishedChunks) { 0849 qCDebug(KGET_DEBUG) << "Redownload everything"; 0850 m_downloadedSize = 0; 0851 } else { 0852 if (offsets.isEmpty()) { 0853 m_startedChunks->clear(); 0854 m_finishedChunks->clear(); 0855 } 0856 qCDebug(KGET_DEBUG) << "Redownload broken pieces"; 0857 for (int i = 0; i < offsets.count(); ++i) { 0858 const int start = offsets[i] / m_segSize; 0859 const int end = std::ceil(length / static_cast<double>(m_segSize)) - 1 + start; 0860 m_startedChunks->setRange(start, end, false); 0861 m_finishedChunks->setRange(start, end, false); 0862 } 0863 0864 m_downloadedSize = m_segSize * m_finishedChunks->numOnBits(); 0865 } 0866 m_prevDownloadedSizes.clear(); 0867 m_prevDownloadedSizes.append(m_downloadedSize); 0868 0869 // remove all current mirrors and readd the first unused mirror 0870 const QList<QUrl> mirrors = 0871 m_sources.keys(); // FIXME only remove the mirrors of the broken segments?! --> for that m_assignedChunks would need to be saved was well 0872 foreach (const QUrl &mirror, mirrors) { 0873 removeMirror(mirror); 0874 } 0875 addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst()); 0876 0877 m_speed = 0; 0878 0879 Transfer::ChangesFlags change = (Transfer::Tc_DownloadSpeed | Transfer::Tc_DownloadedSize); 0880 if (m_size) { 0881 change |= Transfer::Tc_Percent; 0882 m_percent = (m_downloadedSize * 100 / m_size); 0883 } 0884 Q_EMIT dataSourceFactoryChange(change); 0885 m_status = Job::Stopped; 0886 0887 start(); 0888 } 0889 0890 void DataSourceFactory::load(const QDomElement *element) 0891 { 0892 if (!element) { 0893 return; 0894 } 0895 0896 QDomElement e = element->firstChildElement("factory"); 0897 if (e.isNull()) { 0898 e = element->firstChildElement("factories").firstChildElement("factory"); 0899 } 0900 0901 // only try to load values if they haven't been set 0902 if (m_dest.isEmpty()) { 0903 m_dest = QUrl(e.attribute("dest")); 0904 } 0905 0906 verifier()->load(e); 0907 signature()->load(e); 0908 0909 Transfer::ChangesFlags change = Transfer::Tc_None; 0910 0911 if (!m_size) { 0912 m_size = e.attribute("size").toULongLong(); 0913 change |= Transfer::Tc_TotalSize; 0914 } 0915 KIO::fileoffset_t tempSegSize = e.attribute("segmentSize").toLongLong(); 0916 if (tempSegSize) { 0917 m_segSize = tempSegSize; 0918 } 0919 if (!m_downloadedSize) { 0920 m_downloadedSize = e.attribute("processedSize").toULongLong(); 0921 change |= Transfer::Tc_DownloadedSize; 0922 if (m_size) { 0923 m_percent = (m_downloadedSize * 100 / m_size); 0924 change |= Transfer::Tc_Percent; 0925 } 0926 } 0927 if (e.hasAttribute("doDownload")) { 0928 m_doDownload = QVariant(e.attribute("doDownload")).toBool(); 0929 } 0930 if (e.hasAttribute("downloadInitialized")) { 0931 m_downloadInitialized = QVariant(e.attribute("downloadInitialized")).toBool(); 0932 } 0933 if (e.hasAttribute("maxMirrorsUsed")) { 0934 bool worked; 0935 m_maxMirrorsUsed = e.attribute("maxMirrorsUsed").toInt(&worked); 0936 m_maxMirrorsUsed = (worked ? m_maxMirrorsUsed : 3); 0937 } 0938 m_sizeInitiallyDefined = QVariant(e.attribute("sizeInitiallyDefined", "false")).toBool(); 0939 m_sizeFoundOnFinish = QVariant(e.attribute("sizeFoundOnFinish", "false")).toBool(); 0940 0941 // load the finishedChunks 0942 const QDomElement chunks = e.firstChildElement("chunks"); 0943 const QDomNodeList chunkList = chunks.elementsByTagName("chunk"); 0944 0945 const quint32 numBits = chunks.attribute("numBits").toInt(); 0946 const int numBytes = chunks.attribute("numBytes").toInt(); 0947 QVarLengthArray<quint8> data(numBytes); 0948 0949 if (numBytes && (numBytes == chunkList.length())) { 0950 for (int i = 0; i < numBytes; ++i) { 0951 const quint8 value = chunkList.at(i).toElement().text().toInt(); 0952 data[i] = value; 0953 } 0954 0955 if (!m_finishedChunks) { 0956 m_finishedChunks = new BitSet(data.data(), numBits); 0957 qCDebug(KGET_DEBUG) << m_finishedChunks->numOnBits() << " bits on of " << numBits << " bits."; 0958 } 0959 0960 // set the finished chunks to started 0961 if (!m_startedChunks) { 0962 m_startedChunks = new BitSet(data.data(), numBits); 0963 } 0964 } 0965 m_prevDownloadedSizes.clear(); 0966 m_prevDownloadedSizes.append(m_downloadedSize); 0967 0968 init(); 0969 0970 // add the used urls 0971 const QDomNodeList urls = e.firstChildElement("urls").elementsByTagName("url"); 0972 for (int i = 0; i < urls.count(); ++i) { 0973 const QDomElement urlElement = urls.at(i).toElement(); 0974 const QUrl url = QUrl(urlElement.text()); 0975 const int connections = urlElement.attribute("numParallelSegments").toInt(); 0976 addMirror(url, true, connections, true); 0977 } 0978 0979 // add the unused urls 0980 const QDomNodeList unusedUrls = e.firstChildElement("unusedUrls").elementsByTagName("url"); 0981 for (int i = 0; i < unusedUrls.count(); ++i) { 0982 const QDomElement urlElement = unusedUrls.at(i).toElement(); 0983 const QUrl url = QUrl(urlElement.text()); 0984 const int connections = urlElement.attribute("numParallelSegments").toInt(); 0985 addMirror(url, false, connections, true); 0986 } 0987 0988 // m_status = static_cast<Job::Status>(e.attribute("status").toInt()); 0989 0990 if (change != Transfer::Tc_None) { 0991 Q_EMIT dataSourceFactoryChange(change); 0992 } 0993 } 0994 0995 void DataSourceFactory::changeStatus(Job::Status status) 0996 { 0997 Transfer::ChangesFlags change = Transfer::Tc_Status; 0998 m_status = status; 0999 1000 switch (m_status) { 1001 case Job::Aborted: 1002 case Job::Moving: 1003 case Job::Stopped: 1004 m_speed = 0; 1005 change |= Transfer::Tc_DownloadSpeed; 1006 break; 1007 case Job::Running: 1008 break; 1009 case Job::Finished: 1010 m_speed = 0; 1011 m_percent = 100; 1012 1013 if (m_size) { 1014 m_downloadedSize = m_size; 1015 change |= Transfer::Tc_DownloadedSize; 1016 } else if (m_downloadedSize) { 1017 m_sizeFoundOnFinish = true; 1018 m_size = m_downloadedSize; 1019 change |= Transfer::Tc_TotalSize; 1020 } 1021 1022 change |= Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent; 1023 1024 if (Settings::checksumAutomaticVerification() && verifier()->isVerifyable()) { 1025 verifier()->verify(); 1026 } 1027 if (Settings::signatureAutomaticVerification() && signature()->isVerifyable()) { 1028 signature()->verify(); 1029 } 1030 1031 slotUpdateCapabilities(); 1032 break; 1033 default: 1034 // TODO handle Delayed 1035 break; 1036 } 1037 1038 Q_EMIT dataSourceFactoryChange(change); 1039 } 1040 1041 void DataSourceFactory::save(const QDomElement &element) 1042 { 1043 if (element.isNull()) { 1044 return; 1045 } 1046 1047 QDomElement e = element; 1048 QDomDocument doc(e.ownerDocument()); 1049 1050 QDomElement factories = e.firstChildElement("factories"); 1051 if (factories.isNull()) { 1052 factories = doc.createElement("factories"); 1053 } 1054 1055 QDomElement factory = doc.createElement("factory"); 1056 factory.setAttribute("dest", m_dest.url()); 1057 1058 if (!m_finishedChunks || m_sizeFoundOnFinish) { 1059 factory.setAttribute("processedSize", m_downloadedSize); 1060 } 1061 factory.setAttribute("size", m_size); 1062 factory.setAttribute("segmentSize", m_segSize); 1063 factory.setAttribute("status", m_status); 1064 factory.setAttribute("doDownload", m_doDownload); 1065 factory.setAttribute("downloadInitialized", m_downloadInitialized); 1066 factory.setAttribute("maxMirrorsUsed", m_maxMirrorsUsed); 1067 factory.setAttribute("sizeInitiallyDefined", m_sizeInitiallyDefined); 1068 factory.setAttribute("sizeFoundOnFinish", m_sizeFoundOnFinish); 1069 1070 verifier()->save(factory); 1071 signature()->save(factory); 1072 1073 // set the finished chunks, but only if chunks were actually used 1074 if (m_finishedChunks && !m_sizeFoundOnFinish) { 1075 const KIO::fileoffset_t rest = m_size % m_segSize; 1076 // the lastSegsize is rest, but only if there is a rest and it is the last segment of the download 1077 const KIO::fileoffset_t lastSegSize = (rest ? rest : m_segSize); 1078 1079 // not m_downloadedSize is stored, but the bytes that do not have to be redownloaded 1080 const bool lastOn = m_finishedChunks->get(m_finishedChunks->getNumBits() - 1); 1081 factory.setAttribute("processedSize", m_segSize * (m_finishedChunks->numOnBits() - lastOn) + lastOn * lastSegSize); 1082 1083 QDomElement chunks = doc.createElement("chunks"); 1084 chunks.setAttribute("numBits", m_finishedChunks->getNumBits()); 1085 chunks.setAttribute("numBytes", m_finishedChunks->getNumBytes()); 1086 1087 const quint8 *data = m_finishedChunks->getData(); 1088 for (quint32 i = 0; i < m_finishedChunks->getNumBytes(); ++i) { 1089 QDomElement chunk = doc.createElement("chunk"); 1090 QDomText num = doc.createTextNode(QString::number(data[i])); 1091 chunk.setAttribute("number", i); 1092 chunk.appendChild(num); 1093 chunks.appendChild(chunk); 1094 } 1095 factory.appendChild(chunks); 1096 } 1097 1098 // set the used urls 1099 QDomElement urls = doc.createElement("urls"); 1100 QHash<QUrl, TransferDataSource *>::const_iterator it; 1101 QHash<QUrl, TransferDataSource *>::const_iterator itEnd = m_sources.constEnd(); 1102 for (it = m_sources.constBegin(); it != itEnd; ++it) { 1103 QDomElement url = doc.createElement("url"); 1104 const QDomText text = doc.createTextNode(it.key().url()); 1105 url.appendChild(text); 1106 url.setAttribute("numParallelSegments", (*it)->parallelSegments()); 1107 urls.appendChild(url); 1108 factory.appendChild(urls); 1109 } 1110 1111 // set the unused urls 1112 urls = doc.createElement("unusedUrls"); 1113 for (int i = 0; i < m_unusedUrls.count(); ++i) { 1114 QDomElement url = doc.createElement("url"); 1115 const QDomText text = doc.createTextNode(m_unusedUrls.at(i).url()); 1116 url.appendChild(text); 1117 url.setAttribute("numParallelSegments", m_unusedConnections.at(i)); 1118 urls.appendChild(url); 1119 factory.appendChild(urls); 1120 } 1121 1122 factories.appendChild(factory); 1123 e.appendChild(factories); 1124 } 1125 1126 bool DataSourceFactory::isValid() const 1127 { 1128 // FIXME 1129 return true; 1130 bool valid = (m_size && m_segSize && m_dest.isValid() && !m_sources.isEmpty()); 1131 1132 // if the download is finished the it is valid no matter what is set or not 1133 if (m_status == Job::Finished) { 1134 valid = true; 1135 } 1136 qDebug() << m_size << m_segSize << m_dest.isValid() << !m_sources.isEmpty(); 1137 return valid; 1138 } 1139 1140 Verifier *DataSourceFactory::verifier() 1141 { 1142 if (!m_verifier) { 1143 m_verifier = new Verifier(m_dest, this); 1144 } 1145 1146 return m_verifier; 1147 } 1148 1149 Signature *DataSourceFactory::signature() 1150 { 1151 if (!m_signature) { 1152 m_signature = new Signature(m_dest, this); 1153 } 1154 1155 return m_signature; 1156 } 1157 1158 void DataSourceFactory::slotUpdateCapabilities() 1159 { 1160 const Transfer::Capabilities oldCaps = capabilities(); 1161 Transfer::Capabilities newCaps = {}; 1162 1163 if ((status() == Job::Finished) || (status() == Job::Stopped)) { 1164 newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming; 1165 } else { 1166 foreach (TransferDataSource *source, m_sources) { 1167 if (!source->assignedSegments().isEmpty()) { 1168 if (newCaps) { 1169 newCaps &= source->capabilities(); 1170 } else { 1171 newCaps = source->capabilities(); 1172 } 1173 } 1174 } 1175 } 1176 1177 if (newCaps & Transfer::Cap_Resuming) { 1178 newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming; 1179 } 1180 1181 newCaps |= Transfer::Cap_MultipleMirrors; 1182 1183 if (oldCaps != newCaps) { 1184 m_capabilities = newCaps; 1185 Q_EMIT capabilitiesChanged(); 1186 } 1187 } 1188 1189 #include "moc_datasourcefactory.cpp"