File indexing completed on 2025-01-05 03:53:55

 /* ============================================================
  *
  * This file is a part of digiKam project
  * https://www.digikam.org
  *
  * Date        : 2007-03-21
  * Description : Collection scanning to database - scan utilities.
  *
  * SPDX-FileCopyrightText: 2005-2006 by Tom Albers <tomalbers at kde dot nl>
  * SPDX-FileCopyrightText: 2007-2011 by Marcel Wiesweg <marcel dot wiesweg at gmx dot de>
  * SPDX-FileCopyrightText: 2009-2024 by Gilles Caulier <caulier dot gilles at gmail dot com>
  *
  * SPDX-License-Identifier: GPL-2.0-or-later
  *
  * ============================================================ */
 
 #include "collectionscanner_p.h"
 
 namespace Digikam
 {
 
 void CollectionScanner::loadNameFilters()
 {
     if (!d->nameFilters.isEmpty())
     {
         return;
     }
 
     QStringList imageFilter, audioFilter, videoFilter, ignoreDirectory;
 
     CoreDbAccess().db()->getFilterSettings(&imageFilter, &videoFilter, &audioFilter);
     CoreDbAccess().db()->getIgnoreDirectoryFilterSettings(&ignoreDirectory);
 
     // three sets to find category of a file
 
     d->imageFilterSet  = QSet<QString>(imageFilter.begin(),     imageFilter.end());
     d->audioFilterSet  = QSet<QString>(audioFilter.begin(),     audioFilter.end());
     d->videoFilterSet  = QSet<QString>(videoFilter.begin(),     videoFilter.end());
     d->ignoreDirectory = QSet<QString>(ignoreDirectory.begin(), ignoreDirectory.end());
 
     d->nameFilters     = d->imageFilterSet + d->audioFilterSet + d->videoFilterSet;
 }
 
 void CollectionScanner::mainEntryPoint(bool complete)
 {
     loadNameFilters();
     d->recordHistoryIds = !complete;
 }
 
 void CollectionScanner::safelyRemoveAlbums(const QList<int>& albumIds)
 {
     // Remove the items (orphan items, detach them from the album, but keep entries for a certain time)
     // Make album orphan (no album root, keep entries until next application start)
 
     CoreDbAccess access;
     CoreDbTransaction transaction(&access);
 
     Q_FOREACH (int albumId, albumIds)
     {
         QList<qlonglong> ids = access.db()->getItemIDsInAlbum(albumId);
         access.db()->removeItemsFromAlbum(albumId, ids);
         access.db()->makeStaleAlbum(albumId);
         itemsWereRemoved(ids);
     }
 }
 
 int CollectionScanner::checkAlbum(const CollectionLocation& location, const QString& album)
 {
     // get album id if album exists
 
     int albumID = CoreDbAccess().db()->getAlbumForPath(location.id(), album, false);
 
     d->establishedSourceAlbums.remove(albumID);
 
     // create if necessary
 
     if (albumID == -1)
     {
         QFileInfo fi(location.albumRootPath() + album);
         albumID = CoreDbAccess().db()->addAlbum(location.id(), album, QString(), fi.lastModified().date(), QString());
 
         // have album this one was copied from?
 
         if (d->hints)
         {
             CollectionScannerHints::Album src;
             {
                 QReadLocker locker(&d->hints->lock);
                 src = d->hints->albumHints.value(CollectionScannerHints::DstPath(location.id(), album));
             }
 
             if (!src.isNull())
             {
                 //qCDebug(DIGIKAM_DATABASE_LOG) << "Identified album" << src.albumId << "as source of new album" << fi.filePath();
 
                 CoreDbAccess().db()->copyAlbumProperties(src.albumId, albumID);
                 d->establishedSourceAlbums[albumID] = src.albumId;
             }
         }
     }
 
     return albumID;
 }
 
 void CollectionScanner::copyFileProperties(const ItemInfo& source, const ItemInfo& d)
 {
     if (source.isNull() || d.isNull())
     {
         return;
     }
 
     ItemInfo dest(d);
     CoreDbOperationGroup group;
 
     qCDebug(DIGIKAM_DATABASE_LOG) << "Copying properties from" << source.id() << "to" << dest.id();
 
     // Rating, creation dates
 
     DatabaseFields::ItemInformation imageInfoFields = DatabaseFields::Rating       |
                                                       DatabaseFields::CreationDate |
                                                       DatabaseFields::DigitizationDate;
 
     QVariantList imageInfos = CoreDbAccess().db()->getItemInformation(source.id(), imageInfoFields);
 
     if (!imageInfos.isEmpty())
     {
         CoreDbAccess().db()->changeItemInformation(dest.id(), imageInfos, imageInfoFields);
     }
 
     // Copy public tags
 
     Q_FOREACH (int tagId, TagsCache::instance()->publicTags(source.tagIds()))
     {
         dest.setTag(tagId);
     }
 
     // Copy color and pick label
 
     dest.setPickLabel(source.pickLabel());
     dest.setColorLabel(source.colorLabel());
 
     // important: skip other internal tags, such a history tags. Therefore CoreDB::copyImageTags is not to be used.
 
     // GPS data
 
     QVariantList positionData = CoreDbAccess().db()->getItemPosition(source.id(), DatabaseFields::ItemPositionsAll);
 
     if (!positionData.isEmpty())
     {
         CoreDbAccess().db()->addItemPosition(dest.id(), positionData, DatabaseFields::ItemPositionsAll);
     }
 
     // Comments
 
     {
         CoreDbAccess access;
         ItemComments commentsSource(access, source.id());
         ItemComments commentsDest(access, dest.id());
         commentsDest.replaceFrom(commentsSource);
         commentsDest.apply(access);
     }
 
     // Copyright info
 
     ItemCopyright copyrightDest(dest.id());
     copyrightDest.replaceFrom(ItemCopyright(source.id()));
 
     // Image Properties
 
     CoreDbAccess().db()->copyImageProperties(source.id(), dest.id());
 }
 
 void CollectionScanner::itemsWereRemoved(const QList<qlonglong>& removedIds)
 {
     // set time stamp
 
     d->removedItems();
 
     // manage relations
 
     QList<qlonglong> relatedImages = CoreDbAccess().db()->getOneRelatedImageEach(removedIds, DatabaseRelation::DerivedFrom);
     qCDebug(DIGIKAM_DATABASE_LOG) << "Removed items:" << removedIds << "related items:" << relatedImages;
 
     if (d->recordHistoryIds)
     {
         Q_FOREACH (const qlonglong& id, relatedImages)
         {
             d->needTaggingHistorySet << id;
         }
     }
     else
     {
         int needTaggingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needTaggingHistoryGraph());
         CoreDbAccess().db()->addTagsToItems(relatedImages, QList<int>() << needTaggingTag);
     }
 }
 
 int CollectionScanner::countItemsInFolder(const QString& path)
 {
     QDir dir(path);
 
     if (!dir.exists() || !dir.isReadable())
     {
         return 0;
     }
 
     CollectionLocation location = CollectionManager::instance()->locationForPath(path);
 
     if (!location.isNull() && databaseInitialScanDone())
     {
         QString album = CollectionManager::instance()->album(path);
         int albumID   = CoreDbAccess().db()->getAlbumForPath(location.id(), album, false);
 
         if (albumID != -1)
         {
             QPair<int, int> numberPair = CoreDbAccess().db()->
                                          getNumberOfAllItemsAndAlbums(albumID);
 
             if (numberPair.first)
             {
                 return (numberPair.first + numberPair.second);
             }
         }
     }
 
     // Also count the current folder
 
     int items = 1;
 
     QDirIterator it(dir.path(), QDir::Dirs    |
                                 QDir::Files   |
                                 QDir::NoDotAndDotDot,
                                 QDirIterator::Subdirectories);
 
     while (it.hasNext())
     {
         it.next();
         ++items;
     }
 
     return items;
 }
 
 DatabaseItem::Category CollectionScanner::category(const QFileInfo& info)
 {
     QString suffix = info.suffix().toLower();
 
     if      (d->imageFilterSet.contains(suffix))
     {
         return DatabaseItem::Image;
     }
     else if (d->audioFilterSet.contains(suffix))
     {
         return DatabaseItem::Audio;
     }
     else if (d->videoFilterSet.contains(suffix))
     {
         return DatabaseItem::Video;
     }
     else
     {
         return DatabaseItem::Other;
     }
 }
 
 void CollectionScanner::markDatabaseAsScanned()
 {
     CoreDbAccess access;
     access.db()->setSetting(QLatin1String("Scanned"), QDateTime::currentDateTime().toString(Qt::ISODate));
 }
 
 void CollectionScanner::updateRemovedItemsTime()
 {
     // Called after a complete or partial scan finishes, to write the value
     // held in d->removedItemsTime to the database
 
     if (!d->removedItemsTime.isNull())
     {
         CoreDbAccess().db()->setSetting(QLatin1String("RemovedItemsTime"), d->removedItemsTime.toString(Qt::ISODate));
         d->removedItemsTime = QDateTime();
     }
 }
 
 void CollectionScanner::incrementDeleteRemovedCompleteScanCount()
 {
     CoreDbAccess access;
     int count = access.db()->getSetting(QLatin1String("DeleteRemovedCompleteScanCount")).toInt();
     ++count;
     access.db()->setSetting(QLatin1String("DeleteRemovedCompleteScanCount"), QString::number(count));
 }
 
 void CollectionScanner::resetDeleteRemovedSettings()
 {
     CoreDbAccess().db()->setSetting(QLatin1String("RemovedItemsTime"),               QString());
     CoreDbAccess().db()->setSetting(QLatin1String("DeleteRemovedTime"),              QDateTime::currentDateTime().toString(Qt::ISODate));
     CoreDbAccess().db()->setSetting(QLatin1String("DeleteRemovedCompleteScanCount"), QString::number(0));
 }
 
 bool CollectionScanner::checkDeleteRemoved()
 {
     // returns true if removed items shall be deleted
 
     CoreDbAccess access;
 
     // retrieve last time an item was removed (not deleted, but set to status removed)
 
     QString removedItemsTimeString = access.db()->getSetting(QLatin1String("RemovedItemsTime"));
 
     if (removedItemsTimeString.isNull())
     {
         return false;
     }
 
     // retrieve last time removed items were (definitely) deleted from db
 
     QString deleteRemovedTimeString = access.db()->getSetting(QLatin1String("DeleteRemovedTime"));
     QDateTime removedItemsTime, deleteRemovedTime;
 
     if (!removedItemsTimeString.isNull())
     {
         removedItemsTime = QDateTime::fromString(removedItemsTimeString, Qt::ISODate);
     }
 
     if (!deleteRemovedTimeString.isNull())
     {
         deleteRemovedTime = QDateTime::fromString(deleteRemovedTimeString, Qt::ISODate);
     }
 
     QDateTime now     = QDateTime::currentDateTime();
 
     // retrieve number of complete collection scans since the last time that removed items were deleted
 
     int completeScans = access.db()->getSetting(QLatin1String("DeleteRemovedCompleteScanCount")).toInt();
 
     // No removed items? No need to delete any
 
     if (!removedItemsTime.isValid())
     {
         return false;
     }
 
     // give at least a week between removed item deletions
 
     if (deleteRemovedTime.isValid())
     {
         if (deleteRemovedTime.daysTo(now) <= 7)
         {
             return false;
         }
     }
 
     // Now look at time since items were removed, and the number of complete scans
     // since removed items were deleted. Values arbitrarily chosen.
 
     int daysPast = removedItemsTime.daysTo(now);
 
     return (
             ((daysPast > 7)  && (completeScans > 2)) ||
             ((daysPast > 30) && (completeScans > 0)) ||
             (completeScans > 30)
            );
 }
 
 // ------------------------------------------------------------------------------------------
 
 #if 0
 
 void CollectionScanner::scanForStaleAlbums()
 {
     QStringList albumRootPaths = CollectionManager::instance()->allAvailableAlbumRootPaths();
 
     for (QStringList::const_iterator it = albumRootPaths.constBegin() ;
          it != albumRootPaths.constEnd() ; ++it)
     {
         scanForStaleAlbums(*it);
     }
 }
 
 void CollectionScanner::scanForStaleAlbums(const QString& albumRoot)
 {
     Q_UNUSED(albumRoot);
     QList<AlbumShortInfo> albumList = CoreDbAccess().db()->getAlbumShortInfos();
     QList<AlbumShortInfo> toBeDeleted;
 
     QList<AlbumShortInfo>::const_iterator it;
 
     for (it = albumList.constBegin() ; it != albumList.constEnd() ; ++it)
     {
         QFileInfo fileInfo((*it).albumRoot + (*it).url);
 
         if (!fileInfo.exists() || !fileInfo.isDir())
         {
             m_foldersToBeDeleted << (*it);
         }
     }
 }
 
 QStringList CollectionScanner::formattedListOfStaleAlbums()
 {
     QStringList list;
     QList<AlbumShortInfo>::const_iterator it;
 
     for (it = m_foldersToBeDeleted.constBegin() ;
          it != m_foldersToBeDeleted.constEnd() ; ++it)
     {
         list << (*it).url;
     }
 
     return list;
 }
 
 void CollectionScanner::removeStaleAlbums()
 {
     CoreDbAccess access;
     CoreDbTransaction transaction(&access);
     QList<AlbumShortInfo>::const_iterator it;
 
     for (it = m_foldersToBeDeleted.constBegin() ; it != m_foldersToBeDeleted.constEnd() ; ++it)
     {
         qCDebug(DIGIKAM_DATABASE_LOG) << "Removing album " << (*it).albumRoot + QLatin1Char('/') + (*it).url;
         access.db()->deleteAlbum((*it).id);
     }
 }
 
 QStringList CollectionScanner::formattedListOfStaleFiles()
 {
     QStringList listToBeDeleted;
 
     CoreDbAccess access;
     QList<QPair<QString, int> >::const_iterator it;
 
     for (it = m_filesToBeDeleted.constBegin() ; it != m_filesToBeDeleted.constEnd() ; ++it)
     {
         QString location = QLatin1String(" (") + access.db()->getAlbumPath((*it).second) + QLatin1Char(')');
 
         listToBeDeleted.append((*it).first + location);
     }
 
     return listToBeDeleted;
 }
 
 void CollectionScanner::removeStaleFiles()
 {
     CoreDbAccess access;
     CoreDbTransaction transaction(&access);
     QList<QPair<QString, int> >::const_iterator it;
 
     for (it = m_filesToBeDeleted.constBegin() ; it != m_filesToBeDeleted.constEnd() ; ++it)
     {
         qCDebug(DIGIKAM_DATABASE_LOG) << "Removing: " << (*it).first << " in " << (*it).second;
         access.db()->deleteItem( (*it).second, (*it).first );
     }
 }
 
 void CollectionScanner::scanAlbums()
 {
     QStringList albumRootPaths = CollectionManager::instance()->allAvailableAlbumRootPaths();
     int count = 0;
 
     for (QStringList::const_iterator it = albumRootPaths.constBegin() ; it != albumRootPaths.constEnd() ; ++it)
     {
         count += countItemsInFolder(*it);
     }
 
     Q_EMIT totalFilesToScan(count);
 
     for (QStringList::const_iterator it = albumRootPaths.constBegin() ; it != albumRootPaths.constEnd() ; ++it)
     {
         QDir dir(*it);
         QStringList fileList(dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot));
         CoreDbTransaction transaction;
 
         Q_FOREACH (const QString& dir, fileList)
         {
             scanAlbum(*it, QLatin1Char('/') + dir);
         }
     }
 }
 
 void CollectionScanner::scan(const QString& folderPath)
 {
     CollectionManager* const manager = CollectionManager::instance();
     QUrl url;
     url.setPath(folderPath);
     QString albumRoot = manager->albumRootPath(url);
     QString album     = manager->album(url);
 
     if (albumRoot.isNull())
     {
         qCWarning(DIGIKAM_DATABASE_LOG) << "scanAlbums(QString): folder " << folderPath << " not found in collection.";
         return;
     }
 
     scan(albumRoot, album);
 }
 
 void CollectionScanner::scan(const QString& albumRoot, const QString& album)
 {
     // Step one: remove invalid albums
 
     scanForStaleAlbums(albumRoot);
     removeStaleAlbums();
 
     Q_EMIT totalFilesToScan(countItemsInFolder(albumRoot + album));
 
     // Step two: Scan directories
 
     if (album == QLatin1String("/"))
     {
         // Don't scan files under album root, only descend into directories (?)
 
         QDir dir(albumRoot + album);
         QStringList fileList(dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot));
 
         CoreDbTransaction transaction;
 
         for (QStringList::const_iterator fileIt = fileList.constBegin() ; fileIt != fileList.constEnd() ; ++fileIt)
         {
             scanAlbum(albumRoot, QLatin1Char('/') + (*fileIt));
         }
     }
     else
     {
         CoreDbTransaction transaction;
         scanAlbum(albumRoot, album);
     }
 
     // Step three: Remove invalid files
 
     removeStaleFiles();
 }
 
 void CollectionScanner::scanAlbum(const QString& filePath)
 {
     QUrl url;
     url.setPath(filePath);
     scanAlbum(CollectionManager::instance()->albumRootPath(url), CollectionManager::instance()->album(url));
 }
 
 void CollectionScanner::scanAlbum(const QString& albumRoot, const QString& album)
 {
     // + Adds album if it does not yet exist in the db.
     // + Recursively scans subalbums of album.
     // + Adds files if they do not yet exist in the db.
     // + Adds stale files from the db to m_filesToBeDeleted
     // - Does not add stale albums to m_foldersToBeDeleted.
 
     QDir dir( albumRoot + album );
 
     if (!dir.exists() || !dir.isReadable())
     {
         qCWarning(DIGIKAM_DATABASE_LOG) << "Folder does not exist or is not readable: " << dir.path();
 
         return;
     }
 
     Q_EMIT startScanningAlbum(albumRoot, album);
 
     // get album id if album exists
 
     int albumID = CoreDbAccess().db()->getAlbumForPath(albumRoot, album, false);
 
     if (albumID == -1)
     {
         QFileInfo fi(albumRoot + album);
         albumID = CoreDbAccess().db()->addAlbum(albumRoot, album, QString(), fi.lastModified().date(), QString());
     }
 
     QStringList filesInAlbum = CoreDbAccess().db()->getItemNamesInAlbum( albumID );
 
     QSet<QString> filesFoundInDB;
 
     for (QStringList::const_iterator it = filesInAlbum.constBegin() ;
          it != filesInAlbum.constEnd() ; ++it)
     {
         filesFoundInDB << *it;
     }
 
     const QFileInfoList list = dir.entryInfoList(m_nameFilters, QDir::AllDirs | QDir::Files  | QDir::NoDotAndDotDot /*not CaseSensitive*/);
 
     QFileInfoList::const_iterator fi;
 
     for (fi = list.constBegin() ; fi != list.constEnd() ; ++fi)
     {
         if      (fi->isFile())
         {
             if      (filesFoundInDB.contains(fi->fileName()) )
             {
                 filesFoundInDB.remove(fi->fileName());
             }
             else if (fi->completeSuffix() == QLatin1String("digikamtempfile.tmp"))
             {
                 // ignore temp files we created ourselves
                 continue;
             }
             else
             {
                 qCDebug(DIGIKAM_DATABASE_LOG) << "Adding item " << fi->fileName();
                 addItem(albumID, albumRoot, album, fi->fileName());
             }
         }
         else if (fi->isDir())
         {
             scanAlbum( albumRoot, album + QLatin1Char('/') + fi->fileName() );
         }
     }
 
     // Removing items from the db which we did not see on disk.
 
     if (!filesFoundInDB.isEmpty())
     {
         QSetIterator<QString> it(filesFoundInDB);
 
         while (it.hasNext())
         {
             QPair<QString, int> pair(it.next(),albumID);
 
             if (m_filesToBeDeleted.indexOf(pair) == -1)
             {
                 m_filesToBeDeleted << pair;
             }
         }
     }
 
     Q_EMIT finishedScanningAlbum(albumRoot, album, list.count());
 }
 
 void CollectionScanner::updateItemsWithoutDate()
 {
     QStringList urls  = CoreDbAccess().db()->getAllItemURLsWithoutDate();
 
     Q_EMIT totalFilesToScan(urls.count());
 
     QString albumRoot = CoreDbAccess::albumRoot();
 
     {
         CoreDbTransaction transaction;
 
         for (QStringList::const_iterator it = urls.constBegin() ; it != urls.constEnd() ; ++it)
         {
             Q_EMIT scanningFile(*it);
 
             QFileInfo fi(*it);
             QString albumURL = fi.path();
             albumURL         = QDir::cleanPath(albumURL.remove(albumRoot));
             int albumID      = CoreDbAccess().db()->getAlbumForPath(albumRoot, albumURL);
 
             if (albumID <= 0)
             {
                 qCWarning(DIGIKAM_DATABASE_LOG) << "Album ID == -1: " << albumURL;
             }
 
             if (fi.exists())
             {
                 CollectionScanner::updateItemDate(albumID, albumRoot, albumURL, fi.fileName());
             }
             else
             {
                 QPair<QString, int> pair(fi.fileName(), albumID);
 
                 if (m_filesToBeDeleted.indexOf(pair) == -1)
                 {
                     m_filesToBeDeleted << pair;
                 }
             }
         }
     }
 }
 
 int CollectionScanner::countItemsInFolder(const QString& directory)
 {
     int items = 0;
 
     QDir dir(directory);
 
     if (!dir.exists() || !dir.isReadable())
     {
         return 0;
     }
 
     QFileInfoList list = dir.entryInfoList();
 
     items += list.count();
 
     QFileInfoList::const_iterator fi;
 
     for (fi = list.constBegin() ; fi != list.constEnd() ; ++fi)
     {
         if (fi->isDir()                            &&
             (fi->fileName() != QLatin1String(".")) &&
             (fi->fileName() != QLatin1String("..")))
         {
             items += countItemsInFolder( fi->filePath() );
         }
     }
 
     return items;
 }
 
 void CollectionScanner::markDatabaseAsScanned()
 {
     CoreDbAccess access;
     access.db()->setSetting("Scanned", QDateTime::currentDateTime().toString(Qt::ISODate));
 }
 
 // ------------------- Tools ------------------------
 
 void CollectionScanner::addItem(int albumID, const QString& albumRoot,
                                 const QString& album, const QString& fileName)
 {
     CoreDbAccess access;
     addItem(access, albumID, albumRoot, album, fileName);
 }
 
 void CollectionScanner::addItem(Digikam::CoreDbAccess& access, int albumID,
                                 const QString& albumRoot, const QString& album,
                                 const QString& fileName)
 {
     QString filePath = albumRoot + album + QLatin1Char('/') + fileName;
 
     QString     comment;
     QStringList keywords;
     QDateTime   datetime;
     int         rating;
 
     QScopedPointer<DMetadata> metadata(new DMetadata(filePath));
 
     // Try to get comments from image :
     // In first, from standard JPEG comments, or
     // In second, from EXIF comments tag, or
     // In third, from IPTC comments tag.
 
     comment  = metadata->getImageComment();
 
     // Try to get date and time from image :
     // In first, from EXIF date & time tags, or
     // In second, from IPTC date & time tags.
 
     datetime = metadata->getItemDateTime();
 
     // Try to get image rating from IPTC Urgency tag
     // else use file system time stamp.
     rating   = metadata->getItemRating();
 
     if (!datetime.isValid())
     {
         QFileInfo info(filePath);
         datetime = info.lastModified();
     }
 
     // Try to get image tags from IPTC keywords tags.
 
     metadata->getItemTagsPath(keywords);
 
     access.db()->addItem(albumID, fileName, datetime, comment, rating, keywords);
 }
 
 void CollectionScanner::updateItemDate(int albumID, const QString& albumRoot,
                                        const QString& album, const QString& fileName)
 {
     CoreDbAccess access;
     updateItemDate(access, albumID, albumRoot, album, fileName);
 }
 
 void CollectionScanner::updateItemDate(Digikam::CoreDbAccess& access, int albumID,
                                        const QString& albumRoot, const QString& album,
                                        const QString& fileName)
 {
     QString filePath = albumRoot + album + QLatin1Char('/') + fileName;
 
     QDateTime datetime;
 
     QScopedPointer<DMetadata> metadata(new DMetadata(filePath));
 
     // Trying to get date and time from image :
     // In first, from EXIF date & time tags, or
     // In second, from IPTC date & time tags.
 
     datetime = metadata->getItemDateTime();
 
     if (!datetime.isValid())
     {
         QFileInfo info(filePath);
         datetime = info.lastModified();
     }
 
     access.db()->setItemDate(albumID, fileName, datetime);
 }
 
 #endif // 0
 
 } // namespace Digikam