File indexing completed on 2024-05-19 15:15:50

0001 /*
0002     This file is part of KDE
0003     SPDX-FileCopyrightText: 1999-2000 Waldo Bastian <bastian@kde.org>
0004     SPDX-FileCopyrightText: 2009 Andreas Hartmetz <ahartmetz@gmail.com>
0005 
0006     SPDX-License-Identifier: MIT
0007 */
0008 
0009 // KDE HTTP Cache cleanup tool
0010 
0011 #include <cstring>
0012 #include <stdlib.h>
0013 
0014 #include <QDBusConnection>
0015 #include <QDateTime>
0016 #include <QDir>
0017 #include <QElapsedTimer>
0018 #include <QLocalServer>
0019 #include <QLocalSocket>
0020 #include <QString>
0021 
0022 #include <KLocalizedString>
0023 #include <QDebug>
0024 #include <kprotocolmanager.h>
0025 
0026 #include <QCommandLineOption>
0027 #include <QCommandLineParser>
0028 #include <QCryptographicHash>
0029 #include <QDBusError>
0030 #include <QDataStream>
0031 #include <QStandardPaths>
0032 #include <qplatformdefs.h>
0033 
0034 #include "../utils_p.h"
0035 
0036 QDateTime g_currentDate;
0037 int g_maxCacheAge;
0038 qint64 g_maxCacheSize;
0039 
0040 static const char appFullName[] = "org.kio5.kio_http_cache_cleaner";
0041 static const char appName[] = "kio_http_cache_cleaner";
0042 
0043 // !START OF SYNC!
0044 // Keep the following in sync with the cache code in http.cpp
0045 
0046 static const int s_hashedUrlBits = 160; // this number should always be divisible by eight
0047 static const int s_hashedUrlNibbles = s_hashedUrlBits / 4;
0048 static const int s_hashedUrlBytes = s_hashedUrlBits / 8;
0049 
0050 static const char version[] = "A\n";
0051 
0052 // never instantiated, on-disk / wire format only
0053 struct SerializedCacheFileInfo {
0054     // from http.cpp
0055     quint8 version[2];
0056     quint8 compression; // for now fixed to 0
0057     quint8 reserved; // for now; also alignment
0058     static const int useCountOffset = 4;
0059     qint32 useCount;
0060     qint64 servedDate;
0061     qint64 lastModifiedDate;
0062     qint64 expireDate;
0063     qint32 bytesCached;
0064     static const int size = 36;
0065 
0066     QString url;
0067     QString etag;
0068     QString mimeType;
0069     QStringList responseHeaders; // including status response like "HTTP 200 OK"
0070 };
0071 
0072 struct MiniCacheFileInfo {
0073     // data from cache entry file, or from scoreboard file
0074     qint32 useCount;
0075     // from filesystem
0076     QDateTime lastUsedDate;
0077     qint64 sizeOnDisk;
0078     // we want to delete the least "useful" files and we'll have to sort a list for that...
0079     bool operator<(const MiniCacheFileInfo &other) const;
0080     void debugPrint() const
0081     {
0082         // qDebug() << "useCount:" << useCount
0083         //             << "\nlastUsedDate:" << lastUsedDate.toString(Qt::ISODate)
0084         //             << "\nsizeOnDisk:" << sizeOnDisk << '\n';
0085     }
0086 };
0087 
0088 struct CacheFileInfo : MiniCacheFileInfo {
0089     quint8 version[2];
0090     quint8 compression; // for now fixed to 0
0091     quint8 reserved; // for now; also alignment
0092 
0093     QDateTime servedDate;
0094     QDateTime lastModifiedDate;
0095     QDateTime expireDate;
0096     qint32 bytesCached;
0097 
0098     QString baseName;
0099     QString url;
0100     QString etag;
0101     QString mimeType;
0102     QStringList responseHeaders; // including status response like "HTTP 200 OK"
0103 
0104     void prettyPrint() const
0105     {
0106         QTextStream out(stdout, QIODevice::WriteOnly);
0107         out << "File " << baseName << " version " << version[0] << version[1];
0108         out << "\n cached bytes     " << bytesCached << " useCount " << useCount;
0109         out << "\n servedDate       " << servedDate.toString(Qt::ISODate);
0110         out << "\n lastModifiedDate " << lastModifiedDate.toString(Qt::ISODate);
0111         out << "\n expireDate       " << expireDate.toString(Qt::ISODate);
0112         out << "\n entity tag       " << etag;
0113         out << "\n encoded URL      " << url;
0114         out << "\n mimetype         " << mimeType;
0115         out << "\nResponse headers follow...\n";
0116         for (const QString &h : std::as_const(responseHeaders)) {
0117             out << h << '\n';
0118         }
0119     }
0120 };
0121 
0122 bool MiniCacheFileInfo::operator<(const MiniCacheFileInfo &other) const
0123 {
0124     const int thisUseful = useCount / qMax(lastUsedDate.secsTo(g_currentDate), qint64(1));
0125     const int otherUseful = other.useCount / qMax(other.lastUsedDate.secsTo(g_currentDate), qint64(1));
0126     return thisUseful < otherUseful;
0127 }
0128 
0129 bool CacheFileInfoPtrLessThan(const CacheFileInfo *cf1, const CacheFileInfo *cf2)
0130 {
0131     return *cf1 < *cf2;
0132 }
0133 
0134 enum OperationMode {
0135     CleanCache = 0,
0136     DeleteCache,
0137     FileInfo,
0138 };
0139 
0140 static bool readBinaryHeader(const QByteArray &d, CacheFileInfo *fi)
0141 {
0142     if (d.size() < SerializedCacheFileInfo::size) {
0143         // qDebug() << "readBinaryHeader(): file too small?";
0144         return false;
0145     }
0146     QDataStream stream(d);
0147     stream.setVersion(QDataStream::Qt_4_5);
0148 
0149     stream >> fi->version[0];
0150     stream >> fi->version[1];
0151     if (fi->version[0] != version[0] || fi->version[1] != version[1]) {
0152         // qDebug() << "readBinaryHeader(): wrong magic bytes";
0153         return false;
0154     }
0155     stream >> fi->compression;
0156     stream >> fi->reserved;
0157 
0158     stream >> fi->useCount;
0159 
0160     SerializedCacheFileInfo serialized;
0161 
0162     stream >> serialized.servedDate;
0163     fi->servedDate.setSecsSinceEpoch(serialized.servedDate);
0164     stream >> serialized.lastModifiedDate;
0165     fi->lastModifiedDate.setSecsSinceEpoch(serialized.lastModifiedDate);
0166     stream >> serialized.expireDate;
0167     fi->expireDate.setSecsSinceEpoch(serialized.expireDate);
0168 
0169     stream >> fi->bytesCached;
0170     return true;
0171 }
0172 
0173 static QString filenameFromUrl(const QByteArray &url)
0174 {
0175     QCryptographicHash hash(QCryptographicHash::Sha1);
0176     hash.addData(url);
0177     return QString::fromLatin1(hash.result().toHex());
0178 }
0179 
0180 static QString cacheDir()
0181 {
0182     return QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/kio_http");
0183 }
0184 
0185 static QString filePath(const QString &baseName)
0186 {
0187     QString path = Utils::slashAppended(cacheDir());
0188     path += baseName;
0189     return path;
0190 }
0191 
0192 static bool readLineChecked(QIODevice *dev, QByteArray *line)
0193 {
0194     *line = dev->readLine(8192);
0195     // if nothing read or the line didn't fit into 8192 bytes(!)
0196     if (line->isEmpty() || !line->endsWith('\n')) {
0197         return false;
0198     }
0199     // we don't actually want the newline!
0200     line->chop(1);
0201     return true;
0202 }
0203 
0204 static bool readTextHeader(QFile *file, CacheFileInfo *fi, OperationMode mode)
0205 {
0206     bool ok = true;
0207     QByteArray readBuf;
0208 
0209     ok = ok && readLineChecked(file, &readBuf);
0210     fi->url = QString::fromLatin1(readBuf);
0211     if (filenameFromUrl(readBuf) != QFileInfo(*file).baseName()) {
0212         // qDebug() << "You have witnessed a very improbable hash collision!";
0213         return false;
0214     }
0215 
0216     // only read the necessary info for cache cleaning. Saves time and (more importantly) memory.
0217     if (mode != FileInfo) {
0218         return true;
0219     }
0220 
0221     ok = ok && readLineChecked(file, &readBuf);
0222     fi->etag = QString::fromLatin1(readBuf);
0223 
0224     ok = ok && readLineChecked(file, &readBuf);
0225     fi->mimeType = QString::fromLatin1(readBuf);
0226 
0227     // read as long as no error and no empty line found
0228     while (true) {
0229         ok = ok && readLineChecked(file, &readBuf);
0230         if (ok && !readBuf.isEmpty()) {
0231             fi->responseHeaders.append(QString::fromLatin1(readBuf));
0232         } else {
0233             break;
0234         }
0235     }
0236     return ok; // it may still be false ;)
0237 }
0238 
0239 // TODO common include file with http.cpp?
0240 enum CacheCleanerCommand {
0241     InvalidCommand = 0,
0242     CreateFileNotificationCommand,
0243     UpdateFileCommand,
0244 };
0245 
0246 static bool readCacheFile(const QString &baseName, CacheFileInfo *fi, OperationMode mode)
0247 {
0248     QFile file(filePath(baseName));
0249     if (!file.open(QIODevice::ReadOnly)) {
0250         return false;
0251     }
0252     fi->baseName = baseName;
0253 
0254     QByteArray header = file.read(SerializedCacheFileInfo::size);
0255     // do *not* modify/delete the file if we're in file info mode.
0256     if (!(readBinaryHeader(header, fi) && readTextHeader(&file, fi, mode)) && mode != FileInfo) {
0257         // qDebug() << "read(Text|Binary)Header() returned false, deleting file" << baseName;
0258         file.remove();
0259         return false;
0260     }
0261     // get meta-information from the filesystem
0262     QFileInfo fileInfo(file);
0263     fi->lastUsedDate = fileInfo.lastModified();
0264     fi->sizeOnDisk = fileInfo.size();
0265     return true;
0266 }
0267 
0268 class Scoreboard;
0269 
0270 class CacheIndex
0271 {
0272 public:
0273     explicit CacheIndex(const QString &baseName)
0274     {
0275         QByteArray ba = baseName.toLatin1();
0276         const int sz = ba.size();
0277         const char *input = ba.constData();
0278         Q_ASSERT(sz == s_hashedUrlNibbles);
0279 
0280         int translated = 0;
0281         for (int i = 0; i < sz; i++) {
0282             int c = input[i];
0283 
0284             if (c >= '0' && c <= '9') {
0285                 translated |= c - '0';
0286             } else if (c >= 'a' && c <= 'f') {
0287                 translated |= c - 'a' + 10;
0288             } else {
0289                 Q_ASSERT(false);
0290             }
0291 
0292             if (i & 1) {
0293                 // odd index
0294                 m_index[i >> 1] = translated;
0295                 translated = 0;
0296             } else {
0297                 translated = translated << 4;
0298             }
0299         }
0300 
0301         computeHash();
0302     }
0303 
0304     bool operator==(const CacheIndex &other) const
0305     {
0306         const bool isEqual = memcmp(m_index, other.m_index, s_hashedUrlBytes) == 0;
0307         if (isEqual) {
0308             Q_ASSERT(m_hash == other.m_hash);
0309         }
0310         return isEqual;
0311     }
0312 
0313 private:
0314     explicit CacheIndex(const QByteArray &index)
0315     {
0316         Q_ASSERT(index.length() >= s_hashedUrlBytes);
0317         memcpy(m_index, index.constData(), s_hashedUrlBytes);
0318         computeHash();
0319     }
0320 
0321     void computeHash()
0322     {
0323         uint hash = 0;
0324         const int ints = s_hashedUrlBytes / sizeof(uint);
0325         for (int i = 0; i < ints; i++) {
0326             hash ^= reinterpret_cast<uint *>(&m_index[0])[i];
0327         }
0328         if (const int bytesLeft = s_hashedUrlBytes % sizeof(uint)) {
0329             // dead code until a new url hash algorithm or architecture with sizeof(uint) != 4 appears.
0330             // we have the luxury of ignoring endianness because the hash is never written to disk.
0331             // just merge the bits into the hash in some way.
0332             const int offset = ints * sizeof(uint);
0333             for (int i = 0; i < bytesLeft; i++) {
0334                 hash ^= static_cast<uint>(m_index[offset + i]) << (i * 8);
0335             }
0336         }
0337         m_hash = hash;
0338     }
0339 
0340     friend uint qHash(const CacheIndex &);
0341     friend class Scoreboard;
0342 
0343     quint8 m_index[s_hashedUrlBytes]; // packed binary version of the hexadecimal name
0344     uint m_hash;
0345 };
0346 
0347 uint qHash(const CacheIndex &ci)
0348 {
0349     return ci.m_hash;
0350 }
0351 
0352 static CacheCleanerCommand readCommand(const QByteArray &cmd, CacheFileInfo *fi)
0353 {
0354     readBinaryHeader(cmd, fi);
0355     QDataStream stream(cmd);
0356     stream.skipRawData(SerializedCacheFileInfo::size);
0357 
0358     quint32 ret;
0359     stream >> ret;
0360 
0361     QByteArray baseName;
0362     baseName.resize(s_hashedUrlNibbles);
0363     stream.readRawData(baseName.data(), s_hashedUrlNibbles);
0364     Q_ASSERT(stream.atEnd());
0365     fi->baseName = QString::fromLatin1(baseName);
0366 
0367     Q_ASSERT(ret == CreateFileNotificationCommand || ret == UpdateFileCommand);
0368     return static_cast<CacheCleanerCommand>(ret);
0369 }
0370 
0371 // never instantiated, on-disk format only
0372 struct ScoreboardEntry {
0373     // from scoreboard file
0374     quint8 index[s_hashedUrlBytes];
0375     static const int indexSize = s_hashedUrlBytes;
0376     qint32 useCount;
0377     // from scoreboard file, but compared with filesystem to see if scoreboard has current data
0378     qint64 lastUsedDate;
0379     qint32 sizeOnDisk;
0380     static const int size = 36;
0381     // we want to delete the least "useful" files and we'll have to sort a list for that...
0382     bool operator<(const MiniCacheFileInfo &other) const;
0383 };
0384 
0385 class Scoreboard
0386 {
0387 public:
0388     Scoreboard()
0389     {
0390         // read in the scoreboard...
0391         QFile sboard(filePath(QStringLiteral("scoreboard")));
0392         if (sboard.open(QIODevice::ReadOnly)) {
0393             while (true) {
0394                 QByteArray baIndex = sboard.read(ScoreboardEntry::indexSize);
0395                 QByteArray baRest = sboard.read(ScoreboardEntry::size - ScoreboardEntry::indexSize);
0396                 if (baIndex.size() + baRest.size() != ScoreboardEntry::size) {
0397                     break;
0398                 }
0399 
0400                 const QString entryBasename = QString::fromLatin1(baIndex.toHex());
0401                 MiniCacheFileInfo mcfi;
0402                 if (readAndValidateMcfi(baRest, entryBasename, &mcfi)) {
0403                     m_scoreboard.insert(CacheIndex(baIndex), mcfi);
0404                 }
0405             }
0406         }
0407     }
0408 
0409     void writeOut()
0410     {
0411         // write out the scoreboard
0412         QFile sboard(filePath(QStringLiteral("scoreboard")));
0413         if (!sboard.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
0414             return;
0415         }
0416         QDataStream stream(&sboard);
0417 
0418         QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it = m_scoreboard.constBegin();
0419         for (; it != m_scoreboard.constEnd(); ++it) {
0420             const char *indexData = reinterpret_cast<const char *>(it.key().m_index);
0421             stream.writeRawData(indexData, s_hashedUrlBytes);
0422 
0423             stream << it.value().useCount;
0424             stream << it.value().lastUsedDate.toSecsSinceEpoch();
0425             stream << qint32(it.value().sizeOnDisk);
0426         }
0427     }
0428 
0429     bool fillInfo(const QString &baseName, MiniCacheFileInfo *mcfi)
0430     {
0431         QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it = m_scoreboard.constFind(CacheIndex(baseName));
0432         if (it == m_scoreboard.constEnd()) {
0433             return false;
0434         }
0435         *mcfi = it.value();
0436         return true;
0437     }
0438 
0439     qint64 runCommand(const QByteArray &cmd)
0440     {
0441         // execute the command; return number of bytes if a new file was created, zero otherwise.
0442         Q_ASSERT(cmd.size() == 80);
0443         CacheFileInfo fi;
0444         const CacheCleanerCommand ccc = readCommand(cmd, &fi);
0445         QString fileName = filePath(fi.baseName);
0446 
0447         switch (ccc) {
0448         case CreateFileNotificationCommand:
0449             // qDebug() << "CreateNotificationCommand for" << fi.baseName;
0450             if (!readBinaryHeader(cmd, &fi)) {
0451                 return 0;
0452             }
0453             break;
0454 
0455         case UpdateFileCommand: {
0456             // qDebug() << "UpdateFileCommand for" << fi.baseName;
0457             QFile file(fileName);
0458             file.open(QIODevice::ReadWrite);
0459 
0460             CacheFileInfo fiFromDisk;
0461             QByteArray header = file.read(SerializedCacheFileInfo::size);
0462             if (!readBinaryHeader(header, &fiFromDisk) || fiFromDisk.bytesCached != fi.bytesCached) {
0463                 return 0;
0464             }
0465 
0466             // adjust the use count, to make sure that we actually count up. (workers read the file
0467             // asynchronously...)
0468             const quint32 newUseCount = fiFromDisk.useCount + 1;
0469             QByteArray newHeader = cmd.mid(0, SerializedCacheFileInfo::size);
0470             {
0471                 QDataStream stream(&newHeader, QIODevice::ReadWrite);
0472                 stream.skipRawData(SerializedCacheFileInfo::useCountOffset);
0473                 stream << newUseCount;
0474             }
0475 
0476             file.seek(0);
0477             file.write(newHeader);
0478             file.close();
0479 
0480             if (!readBinaryHeader(newHeader, &fi)) {
0481                 return 0;
0482             }
0483             break;
0484         }
0485 
0486         default:
0487             // qDebug() << "received invalid command";
0488             return 0;
0489         }
0490 
0491         QFileInfo fileInfo(fileName);
0492         fi.lastUsedDate = fileInfo.lastModified();
0493         fi.sizeOnDisk = fileInfo.size();
0494         fi.debugPrint();
0495         // a CacheFileInfo is-a MiniCacheFileInfo which enables the following assignment...
0496         add(fi);
0497         // finally, return cache dir growth (only relevant if a file was actually created!)
0498         return ccc == CreateFileNotificationCommand ? fi.sizeOnDisk : 0;
0499     }
0500 
0501     void add(const CacheFileInfo &fi)
0502     {
0503         m_scoreboard[CacheIndex(fi.baseName)] = fi;
0504     }
0505 
0506     void remove(const QString &basename)
0507     {
0508         m_scoreboard.remove(CacheIndex(basename));
0509     }
0510 
0511     // keep memory usage reasonably low - otherwise entries of nonexistent files don't hurt.
0512     void maybeRemoveStaleEntries(const QList<CacheFileInfo *> &fiList)
0513     {
0514         // don't bother when there are a few bogus entries
0515         if (m_scoreboard.count() < fiList.count() + 100) {
0516             return;
0517         }
0518         // qDebug() << "we have too many fake/stale entries, cleaning up...";
0519         QSet<CacheIndex> realFiles;
0520         for (CacheFileInfo *fi : fiList) {
0521             realFiles.insert(CacheIndex(fi->baseName));
0522         }
0523         QHash<CacheIndex, MiniCacheFileInfo>::Iterator it = m_scoreboard.begin();
0524         while (it != m_scoreboard.end()) {
0525             if (realFiles.contains(it.key())) {
0526                 ++it;
0527             } else {
0528                 it = m_scoreboard.erase(it);
0529             }
0530         }
0531     }
0532 
0533 private:
0534     bool readAndValidateMcfi(const QByteArray &rawData, const QString &basename, MiniCacheFileInfo *mcfi)
0535     {
0536         QDataStream stream(rawData);
0537         stream >> mcfi->useCount;
0538         // check those against filesystem
0539         qint64 lastUsedDate;
0540         stream >> lastUsedDate;
0541         mcfi->lastUsedDate.setSecsSinceEpoch(lastUsedDate);
0542 
0543         qint32 sizeOnDisk;
0544         stream >> sizeOnDisk;
0545         mcfi->sizeOnDisk = sizeOnDisk;
0546         // qDebug() << basename << "sizeOnDisk" << mcfi->sizeOnDisk;
0547 
0548         QFileInfo fileInfo(filePath(basename));
0549         if (!fileInfo.exists()) {
0550             return false;
0551         }
0552         bool ok = true;
0553         ok = ok && fileInfo.lastModified() == mcfi->lastUsedDate;
0554         ok = ok && fileInfo.size() == mcfi->sizeOnDisk;
0555         if (!ok) {
0556             // size or last-modified date not consistent with entry file; reload useCount
0557             // note that avoiding to open the file is the whole purpose of the scoreboard - we only
0558             // open the file if we really have to.
0559             QFile entryFile(fileInfo.absoluteFilePath());
0560             if (!entryFile.open(QIODevice::ReadOnly)) {
0561                 return false;
0562             }
0563             if (entryFile.size() < SerializedCacheFileInfo::size) {
0564                 return false;
0565             }
0566             QDataStream stream(&entryFile);
0567             stream.skipRawData(SerializedCacheFileInfo::useCountOffset);
0568 
0569             stream >> mcfi->useCount;
0570             mcfi->lastUsedDate = fileInfo.lastModified();
0571             mcfi->sizeOnDisk = fileInfo.size();
0572             ok = true;
0573         }
0574         return ok;
0575     }
0576 
0577     QHash<CacheIndex, MiniCacheFileInfo> m_scoreboard;
0578 };
0579 
0580 // Keep the above in sync with the cache code in http.cpp
0581 // !END OF SYNC!
0582 
0583 // remove files and directories used by earlier versions of the HTTP cache.
0584 static void removeOldFiles()
0585 {
0586     const char *oldDirs = "0abcdefghijklmnopqrstuvwxyz";
0587     const int n = strlen(oldDirs);
0588     const QString cacheRootDir = filePath(QString());
0589     for (int i = 0; i < n; ++i) {
0590         const QString dirName = QString::fromLatin1(&oldDirs[i], 1);
0591         QDir(cacheRootDir + dirName).removeRecursively();
0592     }
0593     QFile::remove(cacheRootDir + QLatin1String("cleaned"));
0594 }
0595 
0596 class CacheCleaner
0597 {
0598 public:
0599     CacheCleaner(const QDir &cacheDir)
0600         : m_totalSizeOnDisk(0)
0601     {
0602         // qDebug();
0603         m_fileNameList = cacheDir.entryList(QDir::Files);
0604     }
0605 
0606     // Delete some of the files that need to be deleted. Return true when done, false otherwise.
0607     // This makes interleaved cleaning / serving KIO workers possible.
0608     bool processSlice(Scoreboard *scoreboard = nullptr)
0609     {
0610         QElapsedTimer t;
0611         t.start();
0612         // phase one: gather information about cache files
0613         if (!m_fileNameList.isEmpty()) {
0614             while (t.elapsed() < 100 && !m_fileNameList.isEmpty()) {
0615                 QString baseName = m_fileNameList.takeFirst();
0616                 // check if the filename is of the $s_hashedUrlNibbles letters, 0...f type
0617                 if (baseName.length() < s_hashedUrlNibbles) {
0618                     continue;
0619                 }
0620                 bool nameOk = true;
0621                 for (int i = 0; i < s_hashedUrlNibbles && nameOk; i++) {
0622                     QChar c = baseName[i];
0623                     nameOk = (c >= QLatin1Char('0') && c <= QLatin1Char('9')) || (c >= QLatin1Char('a') && c <= QLatin1Char('f'));
0624                 }
0625                 if (!nameOk) {
0626                     continue;
0627                 }
0628                 if (baseName.length() > s_hashedUrlNibbles) {
0629                     if (QFileInfo(filePath(baseName)).lastModified().secsTo(g_currentDate) > 15 * 60) {
0630                         // it looks like a temporary file that hasn't been touched in > 15 minutes...
0631                         QFile::remove(filePath(baseName));
0632                     }
0633                     // the temporary file might still be written to, leave it alone
0634                     continue;
0635                 }
0636 
0637                 CacheFileInfo *fi = new CacheFileInfo();
0638                 fi->baseName = baseName;
0639 
0640                 bool gotInfo = false;
0641                 if (scoreboard) {
0642                     gotInfo = scoreboard->fillInfo(baseName, fi);
0643                 }
0644                 if (!gotInfo) {
0645                     gotInfo = readCacheFile(baseName, fi, CleanCache);
0646                     if (gotInfo && scoreboard) {
0647                         scoreboard->add(*fi);
0648                     }
0649                 }
0650                 if (gotInfo) {
0651                     m_fiList.append(fi);
0652                     m_totalSizeOnDisk += fi->sizeOnDisk;
0653                 } else {
0654                     delete fi;
0655                 }
0656             }
0657             // qDebug() << "total size of cache files is" << m_totalSizeOnDisk;
0658 
0659             if (m_fileNameList.isEmpty()) {
0660                 // final step of phase one
0661                 std::sort(m_fiList.begin(), m_fiList.end(), CacheFileInfoPtrLessThan);
0662             }
0663             return false;
0664         }
0665 
0666         // phase two: delete files until cache is under maximum allowed size
0667 
0668         // TODO: delete files larger than allowed for a single file
0669         while (t.elapsed() < 100) {
0670             if (m_totalSizeOnDisk <= g_maxCacheSize || m_fiList.isEmpty()) {
0671                 // qDebug() << "total size of cache files after cleaning is" << m_totalSizeOnDisk;
0672                 if (scoreboard) {
0673                     scoreboard->maybeRemoveStaleEntries(m_fiList);
0674                     scoreboard->writeOut();
0675                 }
0676                 qDeleteAll(m_fiList);
0677                 m_fiList.clear();
0678                 return true;
0679             }
0680             CacheFileInfo *fi = m_fiList.takeFirst();
0681             QString filename = filePath(fi->baseName);
0682             if (QFile::remove(filename)) {
0683                 m_totalSizeOnDisk -= fi->sizeOnDisk;
0684                 if (scoreboard) {
0685                     scoreboard->remove(fi->baseName);
0686                 }
0687             }
0688             delete fi;
0689         }
0690         return false;
0691     }
0692 
0693 private:
0694     QStringList m_fileNameList;
0695     QList<CacheFileInfo *> m_fiList;
0696     qint64 m_totalSizeOnDisk;
0697 };
0698 
0699 int main(int argc, char **argv)
0700 {
0701     QCoreApplication app(argc, argv);
0702     app.setApplicationVersion(QStringLiteral("5.0"));
0703 
0704     KLocalizedString::setApplicationDomain("kio5");
0705 
0706     QCommandLineParser parser;
0707     parser.addVersionOption();
0708     parser.setApplicationDescription(QCoreApplication::translate("main", "KDE HTTP cache maintenance tool"));
0709     parser.addHelpOption();
0710     parser.addOption(QCommandLineOption(QStringList{QStringLiteral("clear-all")}, QCoreApplication::translate("main", "Empty the cache")));
0711     parser.addOption(QCommandLineOption(QStringList{QStringLiteral("file-info")},
0712                                         QCoreApplication::translate("main", "Display information about cache file"),
0713                                         QStringLiteral("filename")));
0714     parser.process(app);
0715 
0716     OperationMode mode = CleanCache;
0717     if (parser.isSet(QStringLiteral("clear-all"))) {
0718         mode = DeleteCache;
0719     } else if (parser.isSet(QStringLiteral("file-info"))) {
0720         mode = FileInfo;
0721     }
0722 
0723     // file info mode: no scanning of directories, just output info and exit.
0724     if (mode == FileInfo) {
0725         CacheFileInfo fi;
0726         if (!readCacheFile(parser.value(QStringLiteral("file-info")), &fi, mode)) {
0727             return 1;
0728         }
0729         fi.prettyPrint();
0730         return 0;
0731     }
0732 
0733     // make sure we're the only running instance of the cleaner service
0734     if (mode == CleanCache) {
0735         if (!QDBusConnection::sessionBus().isConnected()) {
0736             QDBusError error(QDBusConnection::sessionBus().lastError());
0737             fprintf(stderr, "%s: Could not connect to D-Bus! (%s: %s)\n", appName, qPrintable(error.name()), qPrintable(error.message()));
0738             return 1;
0739         }
0740 
0741         if (!QDBusConnection::sessionBus().registerService(QString::fromLatin1(appFullName))) {
0742             fprintf(stderr, "%s: Already running!\n", appName);
0743             return 0;
0744         }
0745     }
0746 
0747     g_currentDate = QDateTime::currentDateTime();
0748     g_maxCacheAge = KProtocolManager::maxCacheAge();
0749     g_maxCacheSize = mode == DeleteCache ? -1 : KProtocolManager::maxCacheSize() * 1024;
0750 
0751     QString cacheDirName = cacheDir();
0752     QDir().mkpath(cacheDirName);
0753     QDir cacheDir(cacheDirName);
0754     if (!cacheDir.exists()) {
0755         fprintf(stderr, "%s: '%s' does not exist.\n", appName, qPrintable(cacheDirName));
0756         return 0;
0757     }
0758 
0759     removeOldFiles();
0760 
0761     if (mode == DeleteCache) {
0762         QElapsedTimer t;
0763         t.start();
0764         cacheDir.refresh();
0765         // qDebug() << "time to refresh the cacheDir QDir:" << t.elapsed();
0766         CacheCleaner cleaner(cacheDir);
0767         while (!cleaner.processSlice()) { }
0768         QFile::remove(filePath(QStringLiteral("scoreboard")));
0769         return 0;
0770     }
0771 
0772     QLocalServer lServer;
0773     const QString socketFileName = QStandardPaths::writableLocation(QStandardPaths::RuntimeLocation) + QLatin1String("/kio_http_cache_cleaner");
0774     // we need to create the file by opening the socket, otherwise it won't work
0775     QFile::remove(socketFileName);
0776     if (!lServer.listen(socketFileName)) {
0777         qWarning() << "Error listening on" << socketFileName;
0778     }
0779     QList<QLocalSocket *> sockets;
0780     qint64 newBytesCounter = LLONG_MAX; // force cleaner run on startup
0781 
0782     Scoreboard scoreboard;
0783     CacheCleaner *cleaner = nullptr;
0784     while (QDBusConnection::sessionBus().isConnected()) {
0785         g_currentDate = QDateTime::currentDateTime();
0786 
0787         if (!lServer.isListening()) {
0788             return 1;
0789         }
0790         lServer.waitForNewConnection(100);
0791 
0792         while (QLocalSocket *sock = lServer.nextPendingConnection()) {
0793             sock->waitForConnected();
0794             sockets.append(sock);
0795         }
0796 
0797         for (int i = 0; i < sockets.size(); i++) {
0798             QLocalSocket *sock = sockets[i];
0799             if (sock->state() != QLocalSocket::ConnectedState) {
0800                 if (sock->state() != QLocalSocket::UnconnectedState) {
0801                     sock->waitForDisconnected();
0802                 }
0803                 delete sock;
0804                 sockets.removeAll(sock);
0805                 i--;
0806                 continue;
0807             }
0808             sock->waitForReadyRead(0);
0809             while (true) {
0810                 QByteArray recv = sock->read(80);
0811                 if (recv.isEmpty()) {
0812                     break;
0813                 }
0814                 Q_ASSERT(recv.size() == 80);
0815                 newBytesCounter += scoreboard.runCommand(recv);
0816             }
0817         }
0818 
0819         // interleave cleaning with serving KIO workers to reduce "garbage collection pauses"
0820         if (cleaner) {
0821             if (cleaner->processSlice(&scoreboard)) {
0822                 // that was the last slice, done
0823                 delete cleaner;
0824                 cleaner = nullptr;
0825             }
0826         } else if (newBytesCounter > (g_maxCacheSize / 8)) {
0827             cacheDir.refresh();
0828             cleaner = new CacheCleaner(cacheDir);
0829             newBytesCounter = 0;
0830         }
0831     }
0832     return 0;
0833 }