src/abstractfile/abstractfilelisting.cpp

0001 /*
0002    SPDX-FileCopyrightText: 2016 (c) Matthieu Gallien <matthieu_gallien@yahoo.fr>
0003
0004    SPDX-License-Identifier: LGPL-3.0-or-later
0005  */
0006
0007 #include "abstractfilelisting.h"
0008
0009 #include "config-upnp-qt.h"
0010
0011 #include "abstractfile/indexercommon.h"
0012
0013 #include "filescanner.h"
0014 #include "elisa_settings.h"
0015
0016 #include <QThread>
0017 #include <QFileInfo>
0018 #include <QFile>
0019 #include <QDir>
0020 #include <QFileSystemWatcher>
0021 #include <QSet>
0022 #include <QPair>
0023 #include <QAtomicInt>
0024
0025
0026 #include <algorithm>
0027 #include <utility>
0028
0029 class AbstractFileListingPrivate
0030 {
0031 public:
0032
0033     QStringList mAllRootPaths;
0034
0035     QFileSystemWatcher mFileSystemWatcher;
0036
0037     QHash<QString, QUrl> mAllAlbumCover;
0038
0039     QHash<QUrl, QSet<QPair<QUrl, bool>>> mDiscoveredFiles;
0040
0041     FileScanner mFileScanner;
0042
0043     QHash<QUrl, QDateTime> mAllFiles;
0044
0045     QAtomicInt mStopRequest = 0;
0046
0047     int mImportedTracksCount = 0;
0048
0049     int mNewFilesEmitInterval = 1;
0050
0051     bool mHandleNewFiles = true;
0052
0053     bool mWaitEndTrackRemoval = false;
0054
0055     bool mErrorWatchingFileSystemChanges = false;
0056
0057     bool mIsActive = false;
0058
0059 };
0060
0061 AbstractFileListing::AbstractFileListing(QObject *parent) : QObject(parent), d(std::make_unique<AbstractFileListingPrivate>())
0062 {
0063     connect(&d->mFileSystemWatcher, &QFileSystemWatcher::directoryChanged,
0064             this, &AbstractFileListing::directoryChanged);
0065     connect(&d->mFileSystemWatcher, &QFileSystemWatcher::fileChanged,
0066             this, &AbstractFileListing::fileChanged);
0067 }
0068
0069 AbstractFileListing::~AbstractFileListing()
0070 = default;
0071
0072 void AbstractFileListing::init()
0073 {
0074     qCDebug(orgKdeElisaIndexer()) << "AbstractFileListing::init";
0075     d->mIsActive = true;
0076
0077     const bool autoScan = Elisa::ElisaConfiguration::self()->scanAtStartup();
0078     if (autoScan) {
0079         Q_EMIT askRestoredTracks();
0080     }
0081 }
0082
0083 void AbstractFileListing::stop()
0084 {
0085     d->mIsActive = false;
0086
0087     triggerStop();
0088 }
0089
0090 void AbstractFileListing::newTrackFile(const DataTypes::TrackDataType &partialTrack)
0091 {
0092     auto scanFileInfo = QFileInfo(partialTrack.resourceURI().toLocalFile());
0093     const auto &newTrack = scanOneFile(partialTrack.resourceURI(), scanFileInfo, WatchChangedDirectories | WatchChangedFiles);
0094
0095     if (newTrack.isValid() && newTrack != partialTrack) {
0096         Q_EMIT modifyTracksList({newTrack}, d->mAllAlbumCover);
0097     }
0098 }
0099
0100 void AbstractFileListing::restoredTracks(QHash<QUrl, QDateTime> allFiles)
0101 {
0102     executeInit(std::move(allFiles));
0103
0104     refreshContent();
0105 }
0106
0107 void AbstractFileListing::setAllRootPaths(const QStringList &allRootPaths)
0108 {
0109     d->mAllRootPaths = allRootPaths;
0110 }
0111
0112 void AbstractFileListing::databaseFinishedInsertingTracksList()
0113 {
0114 }
0115
0116 void AbstractFileListing::databaseFinishedRemovingTracksList()
0117 {
0118     if (waitEndTrackRemoval()) {
0119         Q_EMIT indexingFinished();
0120         setWaitEndTrackRemoval(false);
0121     }
0122 }
0123
0124 void AbstractFileListing::applicationAboutToQuit()
0125 {
0126     d->mStopRequest = 1;
0127 }
0128
0129 const QStringList &AbstractFileListing::allRootPaths() const
0130 {
0131     return d->mAllRootPaths;
0132 }
0133
0134 bool AbstractFileListing::canHandleRootPaths() const
0135 {
0136     return true;
0137 }
0138
0139 void AbstractFileListing::scanDirectory(DataTypes::ListTrackDataType &newFiles, const QUrl &path, FileSystemWatchingModes watchForFileSystemChanges)
0140 {
0141     if (d->mStopRequest == 1) {
0142         return;
0143     }
0144
0145     QDir rootDirectory(path.toLocalFile());
0146     rootDirectory.refresh();
0147
0148     if (rootDirectory.exists()) {
0149         if (watchForFileSystemChanges & WatchChangedDirectories) {
0150             watchPath(path.toLocalFile());
0151         }
0152     }
0153
0154     auto currentDirectoryListingFiles = d->mDiscoveredFiles[path];
0155
0156     auto currentFilesList = QSet<QUrl>();
0157
0158     rootDirectory.refresh();
0159     const auto entryList = rootDirectory.entryInfoList(QDir::NoDotAndDotDot | QDir::Files | QDir::Dirs);
0160     for (const auto &oneEntry : entryList) {
0161         auto newFilePath = QUrl::fromLocalFile(oneEntry.canonicalFilePath());
0162
0163         if (oneEntry.isDir() || oneEntry.isFile()) {
0164             currentFilesList.insert(newFilePath);
0165         }
0166     }
0167
0168     auto removedTracks = QVector<QPair<QUrl, bool>>();
0169     for (const auto &removedFilePath : currentDirectoryListingFiles) {
0170         auto itFilePath = std::find(currentFilesList.begin(), currentFilesList.end(), removedFilePath.first);
0171
0172         if (itFilePath != currentFilesList.end()) {
0173             continue;
0174         }
0175
0176         removedTracks.push_back(removedFilePath);
0177     }
0178
0179     auto allRemovedTracks = QList<QUrl>();
0180     for (const auto &oneRemovedTrack : removedTracks) {
0181         if (oneRemovedTrack.second) {
0182             allRemovedTracks.push_back(oneRemovedTrack.first);
0183         } else {
0184             removeFile(oneRemovedTrack.first, allRemovedTracks);
0185         }
0186     }
0187     for (const auto &oneRemovedTrack : removedTracks) {
0188         currentDirectoryListingFiles.remove(oneRemovedTrack);
0189         currentDirectoryListingFiles.remove(oneRemovedTrack);
0190     }
0191
0192     if (!allRemovedTracks.isEmpty()) {
0193         Q_EMIT removedTracksList(allRemovedTracks);
0194     }
0195
0196     if (!d->mHandleNewFiles) {
0197         return;
0198     }
0199
0200     for (const auto &newFilePath : currentFilesList) {
0201         QFileInfo oneEntry(newFilePath.toLocalFile());
0202
0203         if (oneEntry.isDir()) {
0204             addFileInDirectory(newFilePath, path, WatchChangedDirectories | WatchChangedFiles);
0205             scanDirectory(newFiles, newFilePath, WatchChangedDirectories | WatchChangedFiles);
0206
0207             if (d->mStopRequest == 1) {
0208                 break;
0209             }
0210
0211             continue;
0212         }
0213         if (!oneEntry.isFile()) {
0214             continue;
0215         }
0216
0217         auto itExistingFile = allFiles().constFind(newFilePath);
0218         if (itExistingFile != allFiles().cend()) {
0219             if (*itExistingFile >= oneEntry.metadataChangeTime()) {
0220                 allFiles().erase(itExistingFile);
0221                 qCDebug(orgKdeElisaIndexer()) << "AbstractFileListing::scanDirectory" << newFilePath << "file not modified since last scan";
0222                 continue;
0223             }
0224         }
0225
0226         auto newTrack = scanOneFile(newFilePath, oneEntry, WatchChangedDirectories | WatchChangedFiles);
0227
0228         if (newTrack.isValid() && d->mStopRequest == 0) {
0229             addCover(newTrack);
0230
0231             addFileInDirectory(newTrack.resourceURI(), path, WatchChangedDirectories | WatchChangedFiles);
0232             newFiles.push_back(newTrack);
0233
0234             ++d->mImportedTracksCount;
0235
0236             if (newFiles.size() > d->mNewFilesEmitInterval && d->mStopRequest == 0) {
0237                 d->mNewFilesEmitInterval = std::min(50, 1 + d->mNewFilesEmitInterval * d->mNewFilesEmitInterval);
0238                 emitNewFiles(newFiles);
0239                 newFiles.clear();
0240             }
0241         } else {
0242             qCDebug(orgKdeElisaIndexer()) << "AbstractFileListing::scanDirectory" << newFilePath << "is not a valid track";
0243         }
0244
0245         if (d->mStopRequest == 1) {
0246             break;
0247         }
0248     }
0249 }
0250
0251 void AbstractFileListing::directoryChanged(const QString &path)
0252 {
0253     const auto directoryEntry = d->mDiscoveredFiles.find(QUrl::fromLocalFile(path));
0254     if (directoryEntry == d->mDiscoveredFiles.end()) {
0255         return;
0256     }
0257
0258     Q_EMIT indexingStarted();
0259
0260     scanDirectoryTree(path);
0261
0262     Q_EMIT indexingFinished();
0263 }
0264
0265 void AbstractFileListing::fileChanged(const QString &modifiedFileName)
0266 {
0267     QFileInfo modifiedFileInfo(modifiedFileName);
0268     auto modifiedFile = QUrl::fromLocalFile(modifiedFileName);
0269
0270     auto modifiedTrack = scanOneFile(modifiedFile, modifiedFileInfo, WatchChangedDirectories | WatchChangedFiles);
0271
0272     if (modifiedTrack.isValid()) {
0273         Q_EMIT modifyTracksList({modifiedTrack}, d->mAllAlbumCover);
0274     }
0275 }
0276
0277 void AbstractFileListing::executeInit(QHash<QUrl, QDateTime> allFiles)
0278 {
0279     d->mAllFiles = std::move(allFiles);
0280 }
0281
0282 void AbstractFileListing::triggerStop()
0283 {
0284 }
0285
0286 void AbstractFileListing::triggerRefreshOfContent()
0287 {
0288     d->mImportedTracksCount = 0;
0289 }
0290
0291 void AbstractFileListing::refreshContent()
0292 {
0293     triggerRefreshOfContent();
0294 }
0295
0296 DataTypes::TrackDataType AbstractFileListing::scanOneFile(const QUrl &scanFile, const QFileInfo &scanFileInfo, FileSystemWatchingModes watchForFileSystemChanges)
0297 {
0298     DataTypes::TrackDataType newTrack;
0299
0300     qCDebug(orgKdeElisaIndexer) << "AbstractFileListing::scanOneFile" << scanFile;
0301
0302     auto localFileName = scanFile.toLocalFile();
0303
0304     if (!d->mFileScanner.shouldScanFile(localFileName)) {
0305         qCDebug(orgKdeElisaIndexer) << "AbstractFileListing::scanOneFile" << "invalid mime type";
0306         return newTrack;
0307     }
0308
0309     if (scanFileInfo.exists()) {
0310         auto itExistingFile = d->mAllFiles.constFind(scanFile);
0311         if (itExistingFile != d->mAllFiles.cend()) {
0312             if (*itExistingFile >= scanFileInfo.metadataChangeTime()) {
0313                 d->mAllFiles.erase(itExistingFile);
0314                 qCDebug(orgKdeElisaIndexer) << "AbstractFileListing::scanOneFile" << "not changed file";
0315                 return newTrack;
0316             }
0317         }
0318     }
0319
0320     newTrack = d->mFileScanner.scanOneFile(scanFile, scanFileInfo);
0321
0322     if (newTrack.isValid() && scanFileInfo.exists()) {
0323         if (watchForFileSystemChanges & WatchChangedFiles) {
0324             watchPath(scanFile.toLocalFile());
0325         }
0326     }
0327
0328     return newTrack;
0329 }
0330
0331 void AbstractFileListing::watchPath(const QString &pathName)
0332 {
0333     if (!d->mFileSystemWatcher.addPath(pathName)) {
0334         qCDebug(orgKdeElisaIndexer) << "AbstractFileListing::watchPath" << "fail for" << pathName;
0335
0336         if (!d->mErrorWatchingFileSystemChanges) {
0337             d->mErrorWatchingFileSystemChanges = true;
0338             Q_EMIT errorWatchingFileSystemChanges();
0339         }
0340     }
0341 }
0342
0343 void AbstractFileListing::addFileInDirectory(const QUrl &newFile, const QUrl &directoryName, FileSystemWatchingModes watchForFileSystemChanges)
0344 {
0345     const auto directoryEntry = d->mDiscoveredFiles.find(directoryName);
0346     if (directoryEntry == d->mDiscoveredFiles.end()) {
0347         if (watchForFileSystemChanges & WatchChangedDirectories) {
0348             watchPath(directoryName.toLocalFile());
0349         }
0350
0351         QDir currentDirectory(directoryName.toLocalFile());
0352         if (currentDirectory.cdUp()) {
0353             const auto parentDirectoryName = currentDirectory.absolutePath();
0354             const auto parentDirectory = QUrl::fromLocalFile(parentDirectoryName);
0355             const auto parentDirectoryEntry = d->mDiscoveredFiles.find(parentDirectory);
0356             if (parentDirectoryEntry == d->mDiscoveredFiles.end()) {
0357                 if (watchForFileSystemChanges & WatchChangedDirectories) {
0358                     watchPath(parentDirectoryName);
0359                 }
0360             }
0361
0362             auto &parentCurrentDirectoryListingFiles = d->mDiscoveredFiles[parentDirectory];
0363
0364             parentCurrentDirectoryListingFiles.insert({directoryName, false});
0365         }
0366     }
0367     auto &currentDirectoryListingFiles = d->mDiscoveredFiles[directoryName];
0368
0369     QFileInfo isAFile(newFile.toLocalFile());
0370     currentDirectoryListingFiles.insert({newFile, isAFile.isFile()});
0371 }
0372
0373 void AbstractFileListing::scanDirectoryTree(const QString &path)
0374 {
0375     auto newFiles = DataTypes::ListTrackDataType();
0376
0377     qCDebug(orgKdeElisaIndexer()) << "AbstractFileListing::scanDirectoryTree" << path;
0378
0379     scanDirectory(newFiles, QUrl::fromLocalFile(path), WatchChangedDirectories | WatchChangedFiles);
0380
0381     if (!newFiles.isEmpty() && d->mStopRequest == 0) {
0382         emitNewFiles(newFiles);
0383     }
0384 }
0385
0386 void AbstractFileListing::setHandleNewFiles(bool handleThem)
0387 {
0388     d->mHandleNewFiles = handleThem;
0389 }
0390
0391 void AbstractFileListing::emitNewFiles(const DataTypes::ListTrackDataType &tracks)
0392 {
0393     Q_EMIT tracksList(tracks, d->mAllAlbumCover);
0394 }
0395
0396 void AbstractFileListing::addCover(const DataTypes::TrackDataType &newTrack)
0397 {
0398     auto itCover = d->mAllAlbumCover.find(newTrack.album());
0399     if (itCover != d->mAllAlbumCover.end()) {
0400         return;
0401     }
0402
0403     auto coverUrl = d->mFileScanner.searchForCoverFile(newTrack.resourceURI().toLocalFile());
0404     if (!coverUrl.isEmpty()) {
0405         d->mAllAlbumCover[newTrack.resourceURI().toString()] = coverUrl;
0406     }
0407 }
0408
0409 void AbstractFileListing::removeDirectory(const QUrl &removedDirectory, QList<QUrl> &allRemovedFiles)
0410 {
0411     const auto itRemovedDirectory = d->mDiscoveredFiles.constFind(removedDirectory);
0412
0413     if (itRemovedDirectory == d->mDiscoveredFiles.cend()) {
0414         return;
0415     }
0416
0417     const auto &currentRemovedDirectory = *itRemovedDirectory;
0418     for (const auto &itFile : currentRemovedDirectory) {
0419         if (itFile.first.isValid() && !itFile.first.isEmpty()) {
0420             removeFile(itFile.first, allRemovedFiles);
0421             if (itFile.second) {
0422                 allRemovedFiles.push_back(itFile.first);
0423             }
0424         }
0425     }
0426
0427     d->mDiscoveredFiles.erase(itRemovedDirectory);
0428 }
0429
0430 void AbstractFileListing::removeFile(const QUrl &oneRemovedTrack, QList<QUrl> &allRemovedFiles)
0431 {
0432     auto itRemovedDirectory = d->mDiscoveredFiles.find(oneRemovedTrack);
0433     if (itRemovedDirectory != d->mDiscoveredFiles.end()) {
0434         removeDirectory(oneRemovedTrack, allRemovedFiles);
0435     }
0436 }
0437
0438 QHash<QUrl, QDateTime> &AbstractFileListing::allFiles()
0439 {
0440     return d->mAllFiles;
0441 }
0442
0443 void AbstractFileListing::checkFilesToRemove()
0444 {
0445     QList<QUrl> allRemovedFiles;
0446
0447     for (auto itFile = d->mAllFiles.begin(); itFile != d->mAllFiles.end(); ++itFile) {
0448         allRemovedFiles.push_back(itFile.key());
0449     }
0450
0451     qCDebug(orgKdeElisaIndexer()) << "AbstractFileListing::checkFilesToRemove" << allRemovedFiles.size();
0452
0453     if (!allRemovedFiles.isEmpty()) {
0454         setWaitEndTrackRemoval(true);
0455         Q_EMIT removedTracksList(allRemovedFiles);
0456     }
0457 }
0458
0459 FileScanner &AbstractFileListing::fileScanner()
0460 {
0461     return d->mFileScanner;
0462 }
0463
0464 bool AbstractFileListing::waitEndTrackRemoval() const
0465 {
0466     return d->mWaitEndTrackRemoval;
0467 }
0468
0469 void AbstractFileListing::setWaitEndTrackRemoval(bool wait)
0470 {
0471     d->mWaitEndTrackRemoval = wait;
0472 }
0473
0474 bool AbstractFileListing::isActive() const
0475 {
0476     return d->mIsActive;
0477 }
0478
0479
0480 #include "moc_abstractfilelisting.cpp"