File indexing completed on 2024-05-12 05:11:15
0001 /* 0002 * SPDX-FileCopyrightText: 2014 Christian Mollekopf <mollekopf@kolabsys.com> 0003 * 0004 * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL 0005 * 0006 */ 0007 #include "collectionindexingjob.h" 0008 #include "abstractindexer.h" 0009 #include <Akonadi/AgentBase> 0010 #include <Akonadi/CollectionFetchJob> 0011 #include <Akonadi/CollectionFetchScope> 0012 #include <Akonadi/CollectionStatistics> 0013 #include <Akonadi/IndexPolicyAttribute> 0014 #include <Akonadi/ItemFetchJob> 0015 #include <Akonadi/ItemFetchScope> 0016 #include <Akonadi/ServerManager> 0017 #include <KLocalizedString> 0018 #include <akonadi_indexer_agent_debug.h> 0019 0020 CollectionIndexingJob::CollectionIndexingJob(Index &index, const Akonadi::Collection &col, const QList<Akonadi::Item::Id> &pending, QObject *parent) 0021 : KJob(parent) 0022 , m_collection(col) 0023 , m_pending(pending) 0024 , m_index(index) 0025 { 0026 } 0027 0028 void CollectionIndexingJob::setFullSync(bool enable) 0029 { 0030 m_fullSync = enable; 0031 } 0032 0033 void CollectionIndexingJob::start() 0034 { 0035 qCDebug(AKONADI_INDEXER_AGENT_LOG); 0036 m_time.start(); 0037 0038 // Fetch collection for statistics 0039 auto job = new Akonadi::CollectionFetchJob(m_collection, Akonadi::CollectionFetchJob::Base); 0040 job->fetchScope().setIncludeStatistics(true); 0041 job->fetchScope().setListFilter(Akonadi::CollectionFetchScope::NoFilter); 0042 job->fetchScope().fetchAttribute<Akonadi::IndexPolicyAttribute>(); 0043 connect(job, &KJob::finished, this, &CollectionIndexingJob::slotOnCollectionFetched); 0044 job->start(); 0045 } 0046 0047 void CollectionIndexingJob::slotOnCollectionFetched(KJob *job) 0048 { 0049 if (job->error()) { 0050 qCWarning(AKONADI_INDEXER_AGENT_LOG) << "Failed to fetch items: " << job->errorString(); 0051 setError(KJob::UserDefinedError); 0052 emitResult(); 0053 return; 0054 } 0055 m_collection = static_cast<Akonadi::CollectionFetchJob *>(job)->collections().at(0); 0056 if (m_collection.isVirtual() 0057 || (m_collection.hasAttribute<Akonadi::IndexPolicyAttribute>() && !m_collection.attribute<Akonadi::IndexPolicyAttribute>()->indexingEnabled())) { 0058 emitResult(); 0059 return; 0060 } 0061 0062 Q_EMIT status(Akonadi::AgentBase::Running, i18n("Indexing collection: %1", m_collection.displayName())); 0063 Q_EMIT percent(0); 0064 0065 if (!m_index.haveIndexerForMimeTypes(m_collection.contentMimeTypes())) { 0066 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "No indexer for collection, skipping"; 0067 emitResult(); 0068 return; 0069 } 0070 0071 if (m_pending.isEmpty()) { 0072 if (!m_fullSync) { 0073 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing complete. Total time: " << m_time.elapsed(); 0074 emitResult(); 0075 return; 0076 } 0077 findUnindexed(); 0078 return; 0079 } 0080 indexItems(m_pending); 0081 } 0082 0083 void CollectionIndexingJob::indexItems(const QList<Akonadi::Item::Id> &itemIds) 0084 { 0085 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "collectionIndexingJob::indexItems(const QList<Akonadi::Item::Id> &itemIds) count " << itemIds.count(); 0086 Akonadi::Item::List items; 0087 items.reserve(itemIds.size()); 0088 for (const Akonadi::Item::Id id : itemIds) { 0089 items << Akonadi::Item(id); 0090 } 0091 0092 auto fetchJob = new Akonadi::ItemFetchJob(items); 0093 fetchJob->fetchScope().fetchFullPayload(true); 0094 fetchJob->fetchScope().setCacheOnly(true); 0095 fetchJob->fetchScope().setIgnoreRetrievalErrors(true); 0096 fetchJob->fetchScope().setFetchRemoteIdentification(false); 0097 fetchJob->fetchScope().setFetchModificationTime(true); 0098 fetchJob->fetchScope().setAncestorRetrieval(Akonadi::ItemFetchScope::Parent); 0099 fetchJob->setDeliveryOption(Akonadi::ItemFetchJob::EmitItemsIndividually); 0100 fetchJob->setProperty("count", items.size()); 0101 fetchJob->setProperty("start", m_time.elapsed()); 0102 m_progressTotal = items.size(); 0103 m_progressCounter = 0; 0104 0105 connect(fetchJob, &Akonadi::ItemFetchJob::itemsReceived, this, &CollectionIndexingJob::slotPendingItemsReceived); 0106 connect(fetchJob, &KJob::result, this, &CollectionIndexingJob::slotPendingIndexed); 0107 fetchJob->start(); 0108 } 0109 0110 void CollectionIndexingJob::slotPendingItemsReceived(const Akonadi::Item::List &items) 0111 { 0112 qCDebug(AKONADI_INDEXER_AGENT_LOG) << " CollectionIndexingJob::slotPendingItemsReceived " << items.count(); 0113 for (const Akonadi::Item &item : items) { 0114 qCDebug(AKONADI_INDEXER_AGENT_LOG) << " void CollectionIndexingJob::slotPendingItemsReceived(const Akonadi::Item::List &items)" << item.id(); 0115 m_index.index(item); 0116 } 0117 m_progressCounter++; 0118 Q_EMIT percent(100.0 * m_progressCounter / m_progressTotal); 0119 } 0120 0121 void CollectionIndexingJob::slotPendingIndexed(KJob *job) 0122 { 0123 qCDebug(AKONADI_INDEXER_AGENT_LOG) << " CollectionIndexingJob::slotPendingIndexed "; 0124 if (job->error()) { 0125 qCWarning(AKONADI_INDEXER_AGENT_LOG) << "Failed to fetch items: " << job->errorString(); 0126 setError(KJob::UserDefinedError); 0127 emitResult(); 0128 return; 0129 } 0130 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexed " << job->property("count").toInt() 0131 << " items in (ms): " << m_time.elapsed() - job->property("start").toInt(); 0132 0133 if (!m_fullSync) { 0134 m_index.scheduleCommit(); 0135 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing complete. Total time: " << m_time.elapsed(); 0136 emitResult(); 0137 return; 0138 } 0139 0140 // We need to commit, otherwise the count is not accurate 0141 m_index.commit(); 0142 0143 const int start = m_time.elapsed(); 0144 const qlonglong indexedItemsCount = m_index.indexedItems(m_collection.id()); 0145 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexed items count took (ms): " << m_time.elapsed() - start; 0146 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "In index: " << indexedItemsCount; 0147 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Number of Items in collection: " << m_collection.statistics().count() << " In collection " << m_collection.id(); 0148 0149 if (m_collection.statistics().count() == indexedItemsCount) { 0150 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Index up to date"; 0151 emitResult(); 0152 return; 0153 } else { 0154 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Need to find unindexed items"; 0155 } 0156 0157 findUnindexed(); 0158 } 0159 0160 void CollectionIndexingJob::findUnindexed() 0161 { 0162 m_indexedItems.clear(); 0163 m_needsIndexing.clear(); 0164 const int start = m_time.elapsed(); 0165 m_index.findIndexed(m_indexedItems, m_collection.id()); 0166 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Found " << m_indexedItems.size() << " indexed items. Took (ms): " << m_time.elapsed() - start 0167 << " collection id :" << m_collection.id(); 0168 0169 auto job = new Akonadi::ItemFetchJob(m_collection, this); 0170 job->fetchScope().fetchFullPayload(false); 0171 job->fetchScope().setCacheOnly(true); 0172 job->fetchScope().setIgnoreRetrievalErrors(true); 0173 job->fetchScope().setFetchRemoteIdentification(false); 0174 job->fetchScope().setFetchModificationTime(false); 0175 job->fetchScope().setAncestorRetrieval(Akonadi::ItemFetchScope::None); 0176 job->setDeliveryOption(Akonadi::ItemFetchJob::EmitItemsIndividually); 0177 0178 connect(job, &Akonadi::ItemFetchJob::itemsReceived, this, &CollectionIndexingJob::slotUnindexedItemsReceived); 0179 connect(job, &KJob::result, this, &CollectionIndexingJob::slotFoundUnindexed); 0180 job->start(); 0181 } 0182 0183 void CollectionIndexingJob::slotUnindexedItemsReceived(const Akonadi::Item::List &items) 0184 { 0185 // qCDebug(AKONADI_INDEXER_AGENT_LOG) << "CollectionIndexingJob::slotUnindexedItemsReceived found number items :"<<items.count(); 0186 for (const Akonadi::Item &item : items) { 0187 if (!m_indexedItems.remove(item.id())) { 0188 m_needsIndexing << item.id(); 0189 } 0190 } 0191 } 0192 0193 void CollectionIndexingJob::slotFoundUnindexed(KJob *job) 0194 { 0195 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "CollectionIndexingJob::slotFoundUnindexed :m_needsIndexing.isEmpty() : " << m_needsIndexing.isEmpty() 0196 << " count :" << m_needsIndexing.count() << " m_reindexingLock :" << m_reindexingLock << "m_collection.id() " 0197 << m_collection.id(); 0198 if (job->error()) { 0199 qCWarning(AKONADI_INDEXER_AGENT_LOG) << "Failed to fetch items: " << job->errorString(); 0200 setError(KJob::UserDefinedError); 0201 emitResult(); 0202 return; 0203 } 0204 0205 if (!m_indexedItems.isEmpty()) { 0206 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Removing no longer existing items: " << m_indexedItems.size(); 0207 m_index.remove(m_indexedItems, m_collection.contentMimeTypes()); 0208 } 0209 if (!m_needsIndexing.isEmpty() && !m_reindexingLock) { 0210 m_reindexingLock = true; // Avoid an endless loop 0211 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Found unindexed: " << m_needsIndexing.size(); 0212 indexItems(m_needsIndexing); 0213 return; 0214 } 0215 0216 m_index.commit(); 0217 qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing complete. Total time: " << m_time.elapsed(); 0218 emitResult(); 0219 } 0220 0221 #include "moc_collectionindexingjob.cpp"