File indexing completed on 2024-04-28 09:43:49
0001 /*********************************************************************** 0002 * SPDX-FileCopyrightText: 2003-2004 Max Howell <max.howell@methylblue.com> 0003 * SPDX-FileCopyrightText: 2008-2009 Martin Sandsmark <martin.sandsmark@kde.org> 0004 * SPDX-FileCopyrightText: 2022 Harald Sitter <sitter@kde.org> 0005 * 0006 * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL 0007 ***********************************************************************/ 0008 0009 #include "localLister.h" 0010 0011 #include "Config.h" 0012 #include "fileTree.h" 0013 #include "filelight_debug.h" 0014 #include "scan.h" 0015 0016 #include <QElapsedTimer> 0017 #include <QGuiApplication> //postEvent() 0018 #include <QSemaphore> 0019 #include <QStorageInfo> 0020 #include <QThreadPool> 0021 0022 #include "directoryIterator.h" 0023 0024 namespace Filelight 0025 { 0026 QStringList LocalLister::s_remoteMounts; 0027 QStringList LocalLister::s_localMounts; 0028 0029 LocalLister::LocalLister(const QString &path, QList<std::shared_ptr<Folder>> *cachedTrees, ScanManager *parent) 0030 : m_path(path) 0031 , m_trees(cachedTrees) 0032 , m_parent(parent) 0033 { 0034 // add empty directories for any mount points that are in the path 0035 // TODO empty directories is not ideal as adds to fileCount incorrectly 0036 0037 QStringList list(Config::instance()->skipList); 0038 if (!Config::instance()->scanAcrossMounts) { 0039 list += s_localMounts; 0040 } 0041 if (!Config::instance()->scanRemoteMounts) { 0042 list += s_remoteMounts; 0043 } 0044 0045 for (const QString &ignorePath : std::as_const(list)) { 0046 if (ignorePath.startsWith(path)) { 0047 QString folderName = ignorePath; 0048 if (!folderName.endsWith(QLatin1Char('/'))) { 0049 folderName += QLatin1Char('/'); 0050 } 0051 m_trees->append(std::make_shared<Folder>(folderName.toLocal8Bit().constData())); 0052 } 0053 } 0054 } 0055 0056 void LocalLister::run() 0057 { 0058 QElapsedTimer timer; 0059 timer.start(); 0060 // recursively scan the requested path 0061 const QByteArray path = m_path.toUtf8(); 0062 auto tree = scan(path, path); 0063 0064 static constexpr auto msToS = 1000; // not worth using std::chrono for this single line 0065 qCDebug(FILELIGHT_LOG) << "Scan completed in" << (timer.elapsed() / msToS); 0066 0067 // delete the list of trees useful for this scan, 0068 // in a successful scan the contents would now be transferred to 'tree' 0069 delete m_trees; 0070 0071 if (m_parent->m_abort) // scan was cancelled 0072 { 0073 qCDebug(FILELIGHT_LOG) << "Scan successfully aborted"; 0074 tree = nullptr; 0075 } 0076 qCDebug(FILELIGHT_LOG) << "Emitting signal to cache results ..."; 0077 Q_EMIT branchCompleted(tree); 0078 qCDebug(FILELIGHT_LOG) << "Thread terminating ..."; 0079 } 0080 0081 std::shared_ptr<Folder> LocalLister::scan(const QByteArray &path, const QByteArray &dirname) 0082 { 0083 auto cwd = std::make_shared<Folder>(dirname.constData()); 0084 QList<QPair<QByteArray, QByteArray>> subDirectories; 0085 0086 for (const auto &entry : DirectoryIterator(path)) { 0087 if (m_parent->m_abort) { 0088 return cwd; 0089 } 0090 0091 if (entry.isSkipable || entry.isDuplicate) { 0092 continue; 0093 } 0094 0095 if (entry.isFile) { 0096 cwd->append(entry.name.constData(), entry.size); 0097 m_parent->m_totalSize += entry.size; 0098 } else if (entry.isDir) { 0099 std::shared_ptr<Folder> d = nullptr; 0100 const QByteArray new_dirname = entry.name + QByteArrayLiteral("/"); 0101 const QByteArray new_path = path + entry.name + '/'; 0102 0103 // check to see if we've scanned this section already 0104 0105 QMutexLocker lock(&m_treeMutex); 0106 QList<std::shared_ptr<Folder>> toRemove; 0107 for (const auto &folder : std::as_const(*m_trees)) { 0108 if (new_path == folder->name8Bit()) { 0109 qCDebug(FILELIGHT_LOG) << "Tree pre-completed: " << folder->decodedName() << folder.get(); 0110 d = folder; 0111 toRemove << folder; 0112 m_parent->m_files += folder->children(); 0113 cwd->append(folder, new_dirname.constData()); 0114 } 0115 } 0116 0117 for (const auto /* hold the folder, we delete it below! */ folder : std::as_const(toRemove)) { 0118 m_trees->removeAll(folder); 0119 } 0120 0121 lock.unlock(); 0122 0123 if (!d) { // then scan 0124 qCDebug(FILELIGHT_LOG) << "Tree fresh" << new_path << new_dirname; 0125 subDirectories.append({new_path, new_dirname}); 0126 } 0127 } 0128 0129 ++m_parent->m_files; 0130 } 0131 0132 // Scan all subdirectories, either in separate threads or immediately, 0133 // depending on how many free threads there are in the threadpool. 0134 // Yes, it isn't optimal, but it's better than nothing and pretty simple. 0135 QList<std::shared_ptr<Folder>> returnedCwds(subDirectories.count()); 0136 QSemaphore semaphore; 0137 for (int i = 0; i < subDirectories.count(); i++) { 0138 std::function<void()> scanSubdir = [this, i, &subDirectories, &semaphore, &returnedCwds]() { 0139 returnedCwds[i] = scan(subDirectories[i].first, subDirectories[i].second); 0140 semaphore.release(1); 0141 }; 0142 // Workaround! Do not pass the function to tryStart and have it internally create a runnable. 0143 // The runnable will have incorrect ref counting resulting in failed assertions inside QThreadPool. 0144 // Instead create the runnable ourselves to hit the code paths with correct ref counting. 0145 // https://bugs.kde.org/show_bug.cgi?id=449688 0146 auto runnable = QRunnable::create(scanSubdir); 0147 if (!QThreadPool::globalInstance()->tryStart(runnable)) { 0148 scanSubdir(); 0149 } 0150 } 0151 semaphore.acquire(subDirectories.count()); 0152 for (const auto &d : std::as_const(returnedCwds)) { 0153 if (d) { // then scan was successful 0154 cwd->append(d); 0155 } 0156 } 0157 0158 std::sort(cwd->files.begin(), cwd->files.end(), [](const auto &a, const auto &b) { 0159 return a->size() > b->size(); 0160 }); 0161 0162 return cwd; 0163 } 0164 0165 void LocalLister::readMounts() 0166 { 0167 const auto volumes = QStorageInfo::mountedVolumes(); 0168 for (const QStorageInfo &storage : volumes) { 0169 if (storage.isRoot()) { 0170 continue; 0171 } 0172 0173 QString path = storage.rootPath(); 0174 if (!path.endsWith(QLatin1Char('/'))) { 0175 path += QLatin1Char('/'); 0176 } 0177 0178 if (Config::instance()->remoteFsTypes.contains(storage.fileSystemType()) && !s_remoteMounts.contains(path)) { 0179 s_remoteMounts.append(path); 0180 } else if (!s_localMounts.contains(path)) { 0181 s_localMounts.append(path); 0182 } 0183 } 0184 0185 qCDebug(FILELIGHT_LOG) << "Found the following remote filesystems: " << s_remoteMounts; 0186 qCDebug(FILELIGHT_LOG) << "Found the following local filesystems: " << s_localMounts; 0187 } 0188 0189 } // namespace Filelight