File indexing completed on 2025-03-23 09:55:25
0001 /* 0002 This file is part of KDE 0003 SPDX-FileCopyrightText: 1999-2000 Waldo Bastian <bastian@kde.org> 0004 SPDX-FileCopyrightText: 2009 Andreas Hartmetz <ahartmetz@gmail.com> 0005 0006 SPDX-License-Identifier: MIT 0007 */ 0008 0009 // KDE HTTP Cache cleanup tool 0010 0011 #include <cstring> 0012 #include <stdlib.h> 0013 0014 #include <QDBusConnection> 0015 #include <QDateTime> 0016 #include <QDir> 0017 #include <QElapsedTimer> 0018 #include <QLocalServer> 0019 #include <QLocalSocket> 0020 #include <QString> 0021 0022 #include <KLocalizedString> 0023 #include <QDebug> 0024 #include <kprotocolmanager.h> 0025 0026 #include <QCommandLineOption> 0027 #include <QCommandLineParser> 0028 #include <QCryptographicHash> 0029 #include <QDBusError> 0030 #include <QDataStream> 0031 #include <QStandardPaths> 0032 #include <qplatformdefs.h> 0033 0034 #include "../utils_p.h" 0035 0036 QDateTime g_currentDate; 0037 int g_maxCacheAge; 0038 qint64 g_maxCacheSize; 0039 0040 static const char appFullName[] = "org.kio5.kio_http_cache_cleaner"; 0041 static const char appName[] = "kio_http_cache_cleaner"; 0042 0043 // !START OF SYNC! 0044 // Keep the following in sync with the cache code in http.cpp 0045 0046 static const int s_hashedUrlBits = 160; // this number should always be divisible by eight 0047 static const int s_hashedUrlNibbles = s_hashedUrlBits / 4; 0048 static const int s_hashedUrlBytes = s_hashedUrlBits / 8; 0049 0050 static const char version[] = "A\n"; 0051 0052 // never instantiated, on-disk / wire format only 0053 struct SerializedCacheFileInfo { 0054 // from http.cpp 0055 quint8 version[2]; 0056 quint8 compression; // for now fixed to 0 0057 quint8 reserved; // for now; also alignment 0058 static const int useCountOffset = 4; 0059 qint32 useCount; 0060 qint64 servedDate; 0061 qint64 lastModifiedDate; 0062 qint64 expireDate; 0063 qint32 bytesCached; 0064 static const int size = 36; 0065 0066 QString url; 0067 QString etag; 0068 QString mimeType; 0069 QStringList responseHeaders; // including status response like "HTTP 200 OK" 0070 }; 0071 0072 struct MiniCacheFileInfo { 0073 // data from cache entry file, or from scoreboard file 0074 qint32 useCount; 0075 // from filesystem 0076 QDateTime lastUsedDate; 0077 qint64 sizeOnDisk; 0078 // we want to delete the least "useful" files and we'll have to sort a list for that... 0079 bool operator<(const MiniCacheFileInfo &other) const; 0080 void debugPrint() const 0081 { 0082 // qDebug() << "useCount:" << useCount 0083 // << "\nlastUsedDate:" << lastUsedDate.toString(Qt::ISODate) 0084 // << "\nsizeOnDisk:" << sizeOnDisk << '\n'; 0085 } 0086 }; 0087 0088 struct CacheFileInfo : MiniCacheFileInfo { 0089 quint8 version[2]; 0090 quint8 compression; // for now fixed to 0 0091 quint8 reserved; // for now; also alignment 0092 0093 QDateTime servedDate; 0094 QDateTime lastModifiedDate; 0095 QDateTime expireDate; 0096 qint32 bytesCached; 0097 0098 QString baseName; 0099 QString url; 0100 QString etag; 0101 QString mimeType; 0102 QStringList responseHeaders; // including status response like "HTTP 200 OK" 0103 0104 void prettyPrint() const 0105 { 0106 QTextStream out(stdout, QIODevice::WriteOnly); 0107 out << "File " << baseName << " version " << version[0] << version[1]; 0108 out << "\n cached bytes " << bytesCached << " useCount " << useCount; 0109 out << "\n servedDate " << servedDate.toString(Qt::ISODate); 0110 out << "\n lastModifiedDate " << lastModifiedDate.toString(Qt::ISODate); 0111 out << "\n expireDate " << expireDate.toString(Qt::ISODate); 0112 out << "\n entity tag " << etag; 0113 out << "\n encoded URL " << url; 0114 out << "\n mimetype " << mimeType; 0115 out << "\nResponse headers follow...\n"; 0116 for (const QString &h : std::as_const(responseHeaders)) { 0117 out << h << '\n'; 0118 } 0119 } 0120 }; 0121 0122 bool MiniCacheFileInfo::operator<(const MiniCacheFileInfo &other) const 0123 { 0124 const int thisUseful = useCount / qMax(lastUsedDate.secsTo(g_currentDate), qint64(1)); 0125 const int otherUseful = other.useCount / qMax(other.lastUsedDate.secsTo(g_currentDate), qint64(1)); 0126 return thisUseful < otherUseful; 0127 } 0128 0129 bool CacheFileInfoPtrLessThan(const CacheFileInfo *cf1, const CacheFileInfo *cf2) 0130 { 0131 return *cf1 < *cf2; 0132 } 0133 0134 enum OperationMode { 0135 CleanCache = 0, 0136 DeleteCache, 0137 FileInfo, 0138 }; 0139 0140 static bool readBinaryHeader(const QByteArray &d, CacheFileInfo *fi) 0141 { 0142 if (d.size() < SerializedCacheFileInfo::size) { 0143 // qDebug() << "readBinaryHeader(): file too small?"; 0144 return false; 0145 } 0146 QDataStream stream(d); 0147 stream.setVersion(QDataStream::Qt_4_5); 0148 0149 stream >> fi->version[0]; 0150 stream >> fi->version[1]; 0151 if (fi->version[0] != version[0] || fi->version[1] != version[1]) { 0152 // qDebug() << "readBinaryHeader(): wrong magic bytes"; 0153 return false; 0154 } 0155 stream >> fi->compression; 0156 stream >> fi->reserved; 0157 0158 stream >> fi->useCount; 0159 0160 SerializedCacheFileInfo serialized; 0161 0162 stream >> serialized.servedDate; 0163 fi->servedDate.setSecsSinceEpoch(serialized.servedDate); 0164 stream >> serialized.lastModifiedDate; 0165 fi->lastModifiedDate.setSecsSinceEpoch(serialized.lastModifiedDate); 0166 stream >> serialized.expireDate; 0167 fi->expireDate.setSecsSinceEpoch(serialized.expireDate); 0168 0169 stream >> fi->bytesCached; 0170 return true; 0171 } 0172 0173 static QString filenameFromUrl(const QByteArray &url) 0174 { 0175 QCryptographicHash hash(QCryptographicHash::Sha1); 0176 hash.addData(url); 0177 return QString::fromLatin1(hash.result().toHex()); 0178 } 0179 0180 static QString cacheDir() 0181 { 0182 return QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/kio_http"); 0183 } 0184 0185 static QString filePath(const QString &baseName) 0186 { 0187 QString path = Utils::slashAppended(cacheDir()); 0188 path += baseName; 0189 return path; 0190 } 0191 0192 static bool readLineChecked(QIODevice *dev, QByteArray *line) 0193 { 0194 *line = dev->readLine(8192); 0195 // if nothing read or the line didn't fit into 8192 bytes(!) 0196 if (line->isEmpty() || !line->endsWith('\n')) { 0197 return false; 0198 } 0199 // we don't actually want the newline! 0200 line->chop(1); 0201 return true; 0202 } 0203 0204 static bool readTextHeader(QFile *file, CacheFileInfo *fi, OperationMode mode) 0205 { 0206 bool ok = true; 0207 QByteArray readBuf; 0208 0209 ok = ok && readLineChecked(file, &readBuf); 0210 fi->url = QString::fromLatin1(readBuf); 0211 if (filenameFromUrl(readBuf) != QFileInfo(*file).baseName()) { 0212 // qDebug() << "You have witnessed a very improbable hash collision!"; 0213 return false; 0214 } 0215 0216 // only read the necessary info for cache cleaning. Saves time and (more importantly) memory. 0217 if (mode != FileInfo) { 0218 return true; 0219 } 0220 0221 ok = ok && readLineChecked(file, &readBuf); 0222 fi->etag = QString::fromLatin1(readBuf); 0223 0224 ok = ok && readLineChecked(file, &readBuf); 0225 fi->mimeType = QString::fromLatin1(readBuf); 0226 0227 // read as long as no error and no empty line found 0228 while (true) { 0229 ok = ok && readLineChecked(file, &readBuf); 0230 if (ok && !readBuf.isEmpty()) { 0231 fi->responseHeaders.append(QString::fromLatin1(readBuf)); 0232 } else { 0233 break; 0234 } 0235 } 0236 return ok; // it may still be false ;) 0237 } 0238 0239 // TODO common include file with http.cpp? 0240 enum CacheCleanerCommand { 0241 InvalidCommand = 0, 0242 CreateFileNotificationCommand, 0243 UpdateFileCommand, 0244 }; 0245 0246 static bool readCacheFile(const QString &baseName, CacheFileInfo *fi, OperationMode mode) 0247 { 0248 QFile file(filePath(baseName)); 0249 if (!file.open(QIODevice::ReadOnly)) { 0250 return false; 0251 } 0252 fi->baseName = baseName; 0253 0254 QByteArray header = file.read(SerializedCacheFileInfo::size); 0255 // do *not* modify/delete the file if we're in file info mode. 0256 if (!(readBinaryHeader(header, fi) && readTextHeader(&file, fi, mode)) && mode != FileInfo) { 0257 // qDebug() << "read(Text|Binary)Header() returned false, deleting file" << baseName; 0258 file.remove(); 0259 return false; 0260 } 0261 // get meta-information from the filesystem 0262 QFileInfo fileInfo(file); 0263 fi->lastUsedDate = fileInfo.lastModified(); 0264 fi->sizeOnDisk = fileInfo.size(); 0265 return true; 0266 } 0267 0268 class Scoreboard; 0269 0270 class CacheIndex 0271 { 0272 public: 0273 explicit CacheIndex(const QString &baseName) 0274 { 0275 QByteArray ba = baseName.toLatin1(); 0276 const int sz = ba.size(); 0277 const char *input = ba.constData(); 0278 Q_ASSERT(sz == s_hashedUrlNibbles); 0279 0280 int translated = 0; 0281 for (int i = 0; i < sz; i++) { 0282 int c = input[i]; 0283 0284 if (c >= '0' && c <= '9') { 0285 translated |= c - '0'; 0286 } else if (c >= 'a' && c <= 'f') { 0287 translated |= c - 'a' + 10; 0288 } else { 0289 Q_ASSERT(false); 0290 } 0291 0292 if (i & 1) { 0293 // odd index 0294 m_index[i >> 1] = translated; 0295 translated = 0; 0296 } else { 0297 translated = translated << 4; 0298 } 0299 } 0300 0301 computeHash(); 0302 } 0303 0304 bool operator==(const CacheIndex &other) const 0305 { 0306 const bool isEqual = memcmp(m_index, other.m_index, s_hashedUrlBytes) == 0; 0307 if (isEqual) { 0308 Q_ASSERT(m_hash == other.m_hash); 0309 } 0310 return isEqual; 0311 } 0312 0313 private: 0314 explicit CacheIndex(const QByteArray &index) 0315 { 0316 Q_ASSERT(index.length() >= s_hashedUrlBytes); 0317 memcpy(m_index, index.constData(), s_hashedUrlBytes); 0318 computeHash(); 0319 } 0320 0321 void computeHash() 0322 { 0323 uint hash = 0; 0324 const int ints = s_hashedUrlBytes / sizeof(uint); 0325 for (int i = 0; i < ints; i++) { 0326 hash ^= reinterpret_cast<uint *>(&m_index[0])[i]; 0327 } 0328 if (const int bytesLeft = s_hashedUrlBytes % sizeof(uint)) { 0329 // dead code until a new url hash algorithm or architecture with sizeof(uint) != 4 appears. 0330 // we have the luxury of ignoring endianness because the hash is never written to disk. 0331 // just merge the bits into the hash in some way. 0332 const int offset = ints * sizeof(uint); 0333 for (int i = 0; i < bytesLeft; i++) { 0334 hash ^= static_cast<uint>(m_index[offset + i]) << (i * 8); 0335 } 0336 } 0337 m_hash = hash; 0338 } 0339 0340 friend uint qHash(const CacheIndex &); 0341 friend class Scoreboard; 0342 0343 quint8 m_index[s_hashedUrlBytes]; // packed binary version of the hexadecimal name 0344 uint m_hash; 0345 }; 0346 0347 uint qHash(const CacheIndex &ci) 0348 { 0349 return ci.m_hash; 0350 } 0351 0352 static CacheCleanerCommand readCommand(const QByteArray &cmd, CacheFileInfo *fi) 0353 { 0354 readBinaryHeader(cmd, fi); 0355 QDataStream stream(cmd); 0356 stream.skipRawData(SerializedCacheFileInfo::size); 0357 0358 quint32 ret; 0359 stream >> ret; 0360 0361 QByteArray baseName; 0362 baseName.resize(s_hashedUrlNibbles); 0363 stream.readRawData(baseName.data(), s_hashedUrlNibbles); 0364 Q_ASSERT(stream.atEnd()); 0365 fi->baseName = QString::fromLatin1(baseName); 0366 0367 Q_ASSERT(ret == CreateFileNotificationCommand || ret == UpdateFileCommand); 0368 return static_cast<CacheCleanerCommand>(ret); 0369 } 0370 0371 // never instantiated, on-disk format only 0372 struct ScoreboardEntry { 0373 // from scoreboard file 0374 quint8 index[s_hashedUrlBytes]; 0375 static const int indexSize = s_hashedUrlBytes; 0376 qint32 useCount; 0377 // from scoreboard file, but compared with filesystem to see if scoreboard has current data 0378 qint64 lastUsedDate; 0379 qint32 sizeOnDisk; 0380 static const int size = 36; 0381 // we want to delete the least "useful" files and we'll have to sort a list for that... 0382 bool operator<(const MiniCacheFileInfo &other) const; 0383 }; 0384 0385 class Scoreboard 0386 { 0387 public: 0388 Scoreboard() 0389 { 0390 // read in the scoreboard... 0391 QFile sboard(filePath(QStringLiteral("scoreboard"))); 0392 if (sboard.open(QIODevice::ReadOnly)) { 0393 while (true) { 0394 QByteArray baIndex = sboard.read(ScoreboardEntry::indexSize); 0395 QByteArray baRest = sboard.read(ScoreboardEntry::size - ScoreboardEntry::indexSize); 0396 if (baIndex.size() + baRest.size() != ScoreboardEntry::size) { 0397 break; 0398 } 0399 0400 const QString entryBasename = QString::fromLatin1(baIndex.toHex()); 0401 MiniCacheFileInfo mcfi; 0402 if (readAndValidateMcfi(baRest, entryBasename, &mcfi)) { 0403 m_scoreboard.insert(CacheIndex(baIndex), mcfi); 0404 } 0405 } 0406 } 0407 } 0408 0409 void writeOut() 0410 { 0411 // write out the scoreboard 0412 QFile sboard(filePath(QStringLiteral("scoreboard"))); 0413 if (!sboard.open(QIODevice::WriteOnly | QIODevice::Truncate)) { 0414 return; 0415 } 0416 QDataStream stream(&sboard); 0417 0418 QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it = m_scoreboard.constBegin(); 0419 for (; it != m_scoreboard.constEnd(); ++it) { 0420 const char *indexData = reinterpret_cast<const char *>(it.key().m_index); 0421 stream.writeRawData(indexData, s_hashedUrlBytes); 0422 0423 stream << it.value().useCount; 0424 stream << it.value().lastUsedDate.toSecsSinceEpoch(); 0425 stream << qint32(it.value().sizeOnDisk); 0426 } 0427 } 0428 0429 bool fillInfo(const QString &baseName, MiniCacheFileInfo *mcfi) 0430 { 0431 QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it = m_scoreboard.constFind(CacheIndex(baseName)); 0432 if (it == m_scoreboard.constEnd()) { 0433 return false; 0434 } 0435 *mcfi = it.value(); 0436 return true; 0437 } 0438 0439 qint64 runCommand(const QByteArray &cmd) 0440 { 0441 // execute the command; return number of bytes if a new file was created, zero otherwise. 0442 Q_ASSERT(cmd.size() == 80); 0443 CacheFileInfo fi; 0444 const CacheCleanerCommand ccc = readCommand(cmd, &fi); 0445 QString fileName = filePath(fi.baseName); 0446 0447 switch (ccc) { 0448 case CreateFileNotificationCommand: 0449 // qDebug() << "CreateNotificationCommand for" << fi.baseName; 0450 if (!readBinaryHeader(cmd, &fi)) { 0451 return 0; 0452 } 0453 break; 0454 0455 case UpdateFileCommand: { 0456 // qDebug() << "UpdateFileCommand for" << fi.baseName; 0457 QFile file(fileName); 0458 file.open(QIODevice::ReadWrite); 0459 0460 CacheFileInfo fiFromDisk; 0461 QByteArray header = file.read(SerializedCacheFileInfo::size); 0462 if (!readBinaryHeader(header, &fiFromDisk) || fiFromDisk.bytesCached != fi.bytesCached) { 0463 return 0; 0464 } 0465 0466 // adjust the use count, to make sure that we actually count up. (workers read the file 0467 // asynchronously...) 0468 const quint32 newUseCount = fiFromDisk.useCount + 1; 0469 QByteArray newHeader = cmd.mid(0, SerializedCacheFileInfo::size); 0470 { 0471 QDataStream stream(&newHeader, QIODevice::ReadWrite); 0472 stream.skipRawData(SerializedCacheFileInfo::useCountOffset); 0473 stream << newUseCount; 0474 } 0475 0476 file.seek(0); 0477 file.write(newHeader); 0478 file.close(); 0479 0480 if (!readBinaryHeader(newHeader, &fi)) { 0481 return 0; 0482 } 0483 break; 0484 } 0485 0486 default: 0487 // qDebug() << "received invalid command"; 0488 return 0; 0489 } 0490 0491 QFileInfo fileInfo(fileName); 0492 fi.lastUsedDate = fileInfo.lastModified(); 0493 fi.sizeOnDisk = fileInfo.size(); 0494 fi.debugPrint(); 0495 // a CacheFileInfo is-a MiniCacheFileInfo which enables the following assignment... 0496 add(fi); 0497 // finally, return cache dir growth (only relevant if a file was actually created!) 0498 return ccc == CreateFileNotificationCommand ? fi.sizeOnDisk : 0; 0499 } 0500 0501 void add(const CacheFileInfo &fi) 0502 { 0503 m_scoreboard[CacheIndex(fi.baseName)] = fi; 0504 } 0505 0506 void remove(const QString &basename) 0507 { 0508 m_scoreboard.remove(CacheIndex(basename)); 0509 } 0510 0511 // keep memory usage reasonably low - otherwise entries of nonexistent files don't hurt. 0512 void maybeRemoveStaleEntries(const QList<CacheFileInfo *> &fiList) 0513 { 0514 // don't bother when there are a few bogus entries 0515 if (m_scoreboard.count() < fiList.count() + 100) { 0516 return; 0517 } 0518 // qDebug() << "we have too many fake/stale entries, cleaning up..."; 0519 QSet<CacheIndex> realFiles; 0520 for (CacheFileInfo *fi : fiList) { 0521 realFiles.insert(CacheIndex(fi->baseName)); 0522 } 0523 QHash<CacheIndex, MiniCacheFileInfo>::Iterator it = m_scoreboard.begin(); 0524 while (it != m_scoreboard.end()) { 0525 if (realFiles.contains(it.key())) { 0526 ++it; 0527 } else { 0528 it = m_scoreboard.erase(it); 0529 } 0530 } 0531 } 0532 0533 private: 0534 bool readAndValidateMcfi(const QByteArray &rawData, const QString &basename, MiniCacheFileInfo *mcfi) 0535 { 0536 QDataStream stream(rawData); 0537 stream >> mcfi->useCount; 0538 // check those against filesystem 0539 qint64 lastUsedDate; 0540 stream >> lastUsedDate; 0541 mcfi->lastUsedDate.setSecsSinceEpoch(lastUsedDate); 0542 0543 qint32 sizeOnDisk; 0544 stream >> sizeOnDisk; 0545 mcfi->sizeOnDisk = sizeOnDisk; 0546 // qDebug() << basename << "sizeOnDisk" << mcfi->sizeOnDisk; 0547 0548 QFileInfo fileInfo(filePath(basename)); 0549 if (!fileInfo.exists()) { 0550 return false; 0551 } 0552 bool ok = true; 0553 ok = ok && fileInfo.lastModified() == mcfi->lastUsedDate; 0554 ok = ok && fileInfo.size() == mcfi->sizeOnDisk; 0555 if (!ok) { 0556 // size or last-modified date not consistent with entry file; reload useCount 0557 // note that avoiding to open the file is the whole purpose of the scoreboard - we only 0558 // open the file if we really have to. 0559 QFile entryFile(fileInfo.absoluteFilePath()); 0560 if (!entryFile.open(QIODevice::ReadOnly)) { 0561 return false; 0562 } 0563 if (entryFile.size() < SerializedCacheFileInfo::size) { 0564 return false; 0565 } 0566 QDataStream stream(&entryFile); 0567 stream.skipRawData(SerializedCacheFileInfo::useCountOffset); 0568 0569 stream >> mcfi->useCount; 0570 mcfi->lastUsedDate = fileInfo.lastModified(); 0571 mcfi->sizeOnDisk = fileInfo.size(); 0572 ok = true; 0573 } 0574 return ok; 0575 } 0576 0577 QHash<CacheIndex, MiniCacheFileInfo> m_scoreboard; 0578 }; 0579 0580 // Keep the above in sync with the cache code in http.cpp 0581 // !END OF SYNC! 0582 0583 // remove files and directories used by earlier versions of the HTTP cache. 0584 static void removeOldFiles() 0585 { 0586 const char *oldDirs = "0abcdefghijklmnopqrstuvwxyz"; 0587 const int n = strlen(oldDirs); 0588 const QString cacheRootDir = filePath(QString()); 0589 for (int i = 0; i < n; ++i) { 0590 const QString dirName = QString::fromLatin1(&oldDirs[i], 1); 0591 QDir(cacheRootDir + dirName).removeRecursively(); 0592 } 0593 QFile::remove(cacheRootDir + QLatin1String("cleaned")); 0594 } 0595 0596 class CacheCleaner 0597 { 0598 public: 0599 CacheCleaner(const QDir &cacheDir) 0600 : m_totalSizeOnDisk(0) 0601 { 0602 // qDebug(); 0603 m_fileNameList = cacheDir.entryList(QDir::Files); 0604 } 0605 0606 // Delete some of the files that need to be deleted. Return true when done, false otherwise. 0607 // This makes interleaved cleaning / serving KIO workers possible. 0608 bool processSlice(Scoreboard *scoreboard = nullptr) 0609 { 0610 QElapsedTimer t; 0611 t.start(); 0612 // phase one: gather information about cache files 0613 if (!m_fileNameList.isEmpty()) { 0614 while (t.elapsed() < 100 && !m_fileNameList.isEmpty()) { 0615 QString baseName = m_fileNameList.takeFirst(); 0616 // check if the filename is of the $s_hashedUrlNibbles letters, 0...f type 0617 if (baseName.length() < s_hashedUrlNibbles) { 0618 continue; 0619 } 0620 bool nameOk = true; 0621 for (int i = 0; i < s_hashedUrlNibbles && nameOk; i++) { 0622 QChar c = baseName[i]; 0623 nameOk = (c >= QLatin1Char('0') && c <= QLatin1Char('9')) || (c >= QLatin1Char('a') && c <= QLatin1Char('f')); 0624 } 0625 if (!nameOk) { 0626 continue; 0627 } 0628 if (baseName.length() > s_hashedUrlNibbles) { 0629 if (QFileInfo(filePath(baseName)).lastModified().secsTo(g_currentDate) > 15 * 60) { 0630 // it looks like a temporary file that hasn't been touched in > 15 minutes... 0631 QFile::remove(filePath(baseName)); 0632 } 0633 // the temporary file might still be written to, leave it alone 0634 continue; 0635 } 0636 0637 CacheFileInfo *fi = new CacheFileInfo(); 0638 fi->baseName = baseName; 0639 0640 bool gotInfo = false; 0641 if (scoreboard) { 0642 gotInfo = scoreboard->fillInfo(baseName, fi); 0643 } 0644 if (!gotInfo) { 0645 gotInfo = readCacheFile(baseName, fi, CleanCache); 0646 if (gotInfo && scoreboard) { 0647 scoreboard->add(*fi); 0648 } 0649 } 0650 if (gotInfo) { 0651 m_fiList.append(fi); 0652 m_totalSizeOnDisk += fi->sizeOnDisk; 0653 } else { 0654 delete fi; 0655 } 0656 } 0657 // qDebug() << "total size of cache files is" << m_totalSizeOnDisk; 0658 0659 if (m_fileNameList.isEmpty()) { 0660 // final step of phase one 0661 std::sort(m_fiList.begin(), m_fiList.end(), CacheFileInfoPtrLessThan); 0662 } 0663 return false; 0664 } 0665 0666 // phase two: delete files until cache is under maximum allowed size 0667 0668 // TODO: delete files larger than allowed for a single file 0669 while (t.elapsed() < 100) { 0670 if (m_totalSizeOnDisk <= g_maxCacheSize || m_fiList.isEmpty()) { 0671 // qDebug() << "total size of cache files after cleaning is" << m_totalSizeOnDisk; 0672 if (scoreboard) { 0673 scoreboard->maybeRemoveStaleEntries(m_fiList); 0674 scoreboard->writeOut(); 0675 } 0676 qDeleteAll(m_fiList); 0677 m_fiList.clear(); 0678 return true; 0679 } 0680 CacheFileInfo *fi = m_fiList.takeFirst(); 0681 QString filename = filePath(fi->baseName); 0682 if (QFile::remove(filename)) { 0683 m_totalSizeOnDisk -= fi->sizeOnDisk; 0684 if (scoreboard) { 0685 scoreboard->remove(fi->baseName); 0686 } 0687 } 0688 delete fi; 0689 } 0690 return false; 0691 } 0692 0693 private: 0694 QStringList m_fileNameList; 0695 QList<CacheFileInfo *> m_fiList; 0696 qint64 m_totalSizeOnDisk; 0697 }; 0698 0699 int main(int argc, char **argv) 0700 { 0701 QCoreApplication app(argc, argv); 0702 app.setApplicationVersion(QStringLiteral("5.0")); 0703 0704 KLocalizedString::setApplicationDomain("kio5"); 0705 0706 QCommandLineParser parser; 0707 parser.addVersionOption(); 0708 parser.setApplicationDescription(QCoreApplication::translate("main", "KDE HTTP cache maintenance tool")); 0709 parser.addHelpOption(); 0710 parser.addOption(QCommandLineOption(QStringList{QStringLiteral("clear-all")}, QCoreApplication::translate("main", "Empty the cache"))); 0711 parser.addOption(QCommandLineOption(QStringList{QStringLiteral("file-info")}, 0712 QCoreApplication::translate("main", "Display information about cache file"), 0713 QStringLiteral("filename"))); 0714 parser.process(app); 0715 0716 OperationMode mode = CleanCache; 0717 if (parser.isSet(QStringLiteral("clear-all"))) { 0718 mode = DeleteCache; 0719 } else if (parser.isSet(QStringLiteral("file-info"))) { 0720 mode = FileInfo; 0721 } 0722 0723 // file info mode: no scanning of directories, just output info and exit. 0724 if (mode == FileInfo) { 0725 CacheFileInfo fi; 0726 if (!readCacheFile(parser.value(QStringLiteral("file-info")), &fi, mode)) { 0727 return 1; 0728 } 0729 fi.prettyPrint(); 0730 return 0; 0731 } 0732 0733 // make sure we're the only running instance of the cleaner service 0734 if (mode == CleanCache) { 0735 if (!QDBusConnection::sessionBus().isConnected()) { 0736 QDBusError error(QDBusConnection::sessionBus().lastError()); 0737 fprintf(stderr, "%s: Could not connect to D-Bus! (%s: %s)\n", appName, qPrintable(error.name()), qPrintable(error.message())); 0738 return 1; 0739 } 0740 0741 if (!QDBusConnection::sessionBus().registerService(QString::fromLatin1(appFullName))) { 0742 fprintf(stderr, "%s: Already running!\n", appName); 0743 return 0; 0744 } 0745 } 0746 0747 g_currentDate = QDateTime::currentDateTime(); 0748 g_maxCacheAge = KProtocolManager::maxCacheAge(); 0749 g_maxCacheSize = mode == DeleteCache ? -1 : KProtocolManager::maxCacheSize() * 1024; 0750 0751 QString cacheDirName = cacheDir(); 0752 QDir().mkpath(cacheDirName); 0753 QDir cacheDir(cacheDirName); 0754 if (!cacheDir.exists()) { 0755 fprintf(stderr, "%s: '%s' does not exist.\n", appName, qPrintable(cacheDirName)); 0756 return 0; 0757 } 0758 0759 removeOldFiles(); 0760 0761 if (mode == DeleteCache) { 0762 QElapsedTimer t; 0763 t.start(); 0764 cacheDir.refresh(); 0765 // qDebug() << "time to refresh the cacheDir QDir:" << t.elapsed(); 0766 CacheCleaner cleaner(cacheDir); 0767 while (!cleaner.processSlice()) { } 0768 QFile::remove(filePath(QStringLiteral("scoreboard"))); 0769 return 0; 0770 } 0771 0772 QLocalServer lServer; 0773 const QString socketFileName = QStandardPaths::writableLocation(QStandardPaths::RuntimeLocation) + QLatin1String("/kio_http_cache_cleaner"); 0774 // we need to create the file by opening the socket, otherwise it won't work 0775 QFile::remove(socketFileName); 0776 if (!lServer.listen(socketFileName)) { 0777 qWarning() << "Error listening on" << socketFileName; 0778 } 0779 QList<QLocalSocket *> sockets; 0780 qint64 newBytesCounter = LLONG_MAX; // force cleaner run on startup 0781 0782 Scoreboard scoreboard; 0783 CacheCleaner *cleaner = nullptr; 0784 while (QDBusConnection::sessionBus().isConnected()) { 0785 g_currentDate = QDateTime::currentDateTime(); 0786 0787 if (!lServer.isListening()) { 0788 return 1; 0789 } 0790 lServer.waitForNewConnection(100); 0791 0792 while (QLocalSocket *sock = lServer.nextPendingConnection()) { 0793 sock->waitForConnected(); 0794 sockets.append(sock); 0795 } 0796 0797 for (int i = 0; i < sockets.size(); i++) { 0798 QLocalSocket *sock = sockets[i]; 0799 if (sock->state() != QLocalSocket::ConnectedState) { 0800 if (sock->state() != QLocalSocket::UnconnectedState) { 0801 sock->waitForDisconnected(); 0802 } 0803 delete sock; 0804 sockets.removeAll(sock); 0805 i--; 0806 continue; 0807 } 0808 sock->waitForReadyRead(0); 0809 while (true) { 0810 QByteArray recv = sock->read(80); 0811 if (recv.isEmpty()) { 0812 break; 0813 } 0814 Q_ASSERT(recv.size() == 80); 0815 newBytesCounter += scoreboard.runCommand(recv); 0816 } 0817 } 0818 0819 // interleave cleaning with serving KIO workers to reduce "garbage collection pauses" 0820 if (cleaner) { 0821 if (cleaner->processSlice(&scoreboard)) { 0822 // that was the last slice, done 0823 delete cleaner; 0824 cleaner = nullptr; 0825 } 0826 } else if (newBytesCounter > (g_maxCacheSize / 8)) { 0827 cacheDir.refresh(); 0828 cleaner = new CacheCleaner(cacheDir); 0829 newBytesCounter = 0; 0830 } 0831 } 0832 return 0; 0833 }