akonadi-search/agent/agent.cpp

0001 /*
0002  * This file is part of the KDE Akonadi Search Project
0003  * SPDX-FileCopyrightText: 2013 Vishesh Handa <me@vhanda.in>
0004  * SPDX-FileCopyrightText: 2014 Christian Mollekopf <mollekopf@kolabsys.com>
0005  *
0006  * SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
0007  *
0008  */
0009
0010 #include "agent.h"
0011
0012 #include "akonotesindexer.h"
0013 #include "calendarindexer.h"
0014 #include "collectionupdatejob.h"
0015 #include "contactindexer.h"
0016 #include "emailindexer.h"
0017 #include "indexeradaptor.h"
0018
0019 #include "priority.h"
0020
0021 #include <Akonadi/AttributeFactory>
0022 #include <Akonadi/ChangeRecorder>
0023 #include <Akonadi/CollectionFetchScope>
0024 #include <Akonadi/EntityDisplayAttribute>
0025 #include <Akonadi/IndexPolicyAttribute>
0026 #include <Akonadi/ItemFetchScope>
0027
0028 #include <Akonadi/AgentManager>
0029 #include <Akonadi/ServerManager>
0030
0031 #include "akonadi_indexer_agent_debug.h"
0032 #include <KConfig>
0033 #include <KConfigGroup>
0034 #include <KLocalizedString>
0035
0036 #define INDEXING_AGENT_VERSION 5
0037
0038 AkonadiIndexingAgent::AkonadiIndexingAgent(const QString &id)
0039     : AgentBase(id)
0040     , m_scheduler(m_index, config(), QSharedPointer<JobFactory>(new JobFactory))
0041 {
0042     lowerIOPriority();
0043     lowerSchedulingPriority();
0044     lowerPriority();
0045
0046     Akonadi::AttributeFactory::registerAttribute<Akonadi::IndexPolicyAttribute>();
0047
0048     KConfigGroup cfg = config()->group(QStringLiteral("General"));
0049     const int agentIndexingVersion = cfg.readEntry("agentIndexingVersion", 0);
0050     bool respectDiacriticAndAccents = cfg.readEntry("respectDiacriticAndAccents", true);
0051     if (agentIndexingVersion < INDEXING_AGENT_VERSION) {
0052         m_index.removeDatabase();
0053         // Don't respect Diacritic and Accents in new Database so search will be more easy.
0054         respectDiacriticAndAccents = false;
0055         QTimer::singleShot(0, &m_scheduler, &Scheduler::scheduleCompleteSync);
0056         cfg.writeEntry("agentIndexingVersion", INDEXING_AGENT_VERSION);
0057         cfg.writeEntry("respectDiacriticAndAccents", respectDiacriticAndAccents);
0058         cfg.sync();
0059     }
0060     m_index.setRespectDiacriticAndAccents(respectDiacriticAndAccents);
0061     if (!m_index.createIndexers()) {
0062         Q_EMIT status(Broken, i18nc("@info:status", "No indexers available"));
0063         setOnline(false);
0064     } else {
0065         setOnline(true);
0066     }
0067     connect(this, &Akonadi::AgentBase::abortRequested, this, &AkonadiIndexingAgent::onAbortRequested);
0068     connect(this, &Akonadi::AgentBase::onlineChanged, this, &AkonadiIndexingAgent::onOnlineChanged);
0069
0070     connect(&m_scheduler, SIGNAL(status(int, QString)), this, SIGNAL(status(int, QString)));
0071     connect(&m_scheduler, &Scheduler::percent, this, &Akonadi::AgentBase::percent);
0072     connect(&m_scheduler, &Scheduler::collectionIndexingFinished, this, &AkonadiIndexingAgent::collectionIndexingFinished);
0073
0074     changeRecorder()->setAllMonitored(true);
0075     changeRecorder()->itemFetchScope().setCacheOnly(true);
0076     changeRecorder()->itemFetchScope().setAncestorRetrieval(Akonadi::ItemFetchScope::Parent);
0077     changeRecorder()->itemFetchScope().setFetchRemoteIdentification(false);
0078     changeRecorder()->itemFetchScope().setFetchModificationTime(false);
0079     changeRecorder()->itemFetchScope().fetchFullPayload(true);
0080     changeRecorder()->collectionFetchScope().fetchAttribute<Akonadi::IndexPolicyAttribute>();
0081     changeRecorder()->collectionFetchScope().setAncestorRetrieval(Akonadi::CollectionFetchScope::All);
0082     changeRecorder()->collectionFetchScope().ancestorFetchScope().fetchAttribute<Akonadi::EntityDisplayAttribute>();
0083     changeRecorder()->collectionFetchScope().setListFilter(Akonadi::CollectionFetchScope::Index);
0084     changeRecorder()->setChangeRecordingEnabled(false);
0085     changeRecorder()->fetchCollection(true);
0086     changeRecorder()->setExclusive(true);
0087
0088     new IndexerAdaptor(this);
0089
0090     // Cleanup agentsrc after migration to 4.13/KF6
0091     Akonadi::AgentManager *agentManager = Akonadi::AgentManager::self();
0092     const Akonadi::AgentInstance::List allAgents = agentManager->instances();
0093     // Cannot use agentManager->instance(oldInstanceName) here, it wouldn't find broken instances.
0094     for (const Akonadi::AgentInstance &inst : allAgents) {
0095         if (inst.identifier() == QLatin1StringView("akonadi_nepomuk_feeder")) {
0096             qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Removing old nepomuk feeder" << inst.identifier();
0097             agentManager->removeInstance(inst);
0098         } else if (inst.identifier() == QLatin1StringView("akonadi_baloo_indexer")) {
0099             qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Removing old Baloo indexer" << inst.identifier();
0100             agentManager->removeInstance(inst);
0101         }
0102     }
0103 }
0104
0105 AkonadiIndexingAgent::~AkonadiIndexingAgent() = default;
0106
0107 void AkonadiIndexingAgent::reindexAll()
0108 {
0109     qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Reindexing everything";
0110     m_scheduler.abort();
0111     m_index.removeDatabase();
0112     m_index.createIndexers();
0113     QTimer::singleShot(0, &m_scheduler, &Scheduler::scheduleCompleteSync);
0114 }
0115
0116 void AkonadiIndexingAgent::reindexCollection(const qlonglong id)
0117 {
0118     qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Reindexing collection " << id;
0119     m_scheduler.scheduleCollection(Akonadi::Collection(id), true);
0120 }
0121
0122 void AkonadiIndexingAgent::reindexCollections(const QList<qlonglong> &ids)
0123 {
0124     qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Reindexing collections " << ids;
0125     for (qlonglong id : ids) {
0126         m_scheduler.scheduleCollection(Akonadi::Collection(id), true);
0127     }
0128 }
0129
0130 qlonglong AkonadiIndexingAgent::indexedItems(const qlonglong id)
0131 {
0132     return m_index.indexedItems(id);
0133 }
0134
0135 void AkonadiIndexingAgent::itemAdded(const Akonadi::Item &item, const Akonadi::Collection &collection)
0136 {
0137     if (!shouldIndex(collection)) {
0138         return;
0139     }
0140
0141     m_scheduler.addItem(item);
0142 }
0143
0144 void AkonadiIndexingAgent::itemChanged(const Akonadi::Item &item, const QSet<QByteArray> &partIdentifiers)
0145 {
0146     if (!shouldIndex(item)) {
0147         return;
0148     }
0149
0150     // We don't index certain parts so we don't care when they change
0151     QSet<QByteArray> pi = partIdentifiers;
0152     QMutableSetIterator<QByteArray> it(pi);
0153     while (it.hasNext()) {
0154         it.next();
0155         if (!it.value().startsWith("PLD:")) {
0156             it.remove();
0157         }
0158     }
0159
0160     if (pi.isEmpty()) {
0161         return;
0162     }
0163     m_scheduler.addItem(item);
0164 }
0165
0166 void AkonadiIndexingAgent::itemsFlagsChanged(const Akonadi::Item::List &items, const QSet<QByteArray> &addedFlags, const QSet<QByteArray> &removedFlags)
0167 {
0168     // We optimize and skip the "shouldIndex" call for each item here, since it's
0169     // cheaper to just let Xapian throw an exception for items that were not
0170     // indexed.
0171     // In most cases the entire batch comes from the same collection, so we will
0172     // only suffer penalty in case of larger batches from non-indexed collections,
0173     // but we assume that that's a much less common case than collections with
0174     // indexing enabled.
0175
0176     // Akonadi always sends batch of items of the same type
0177     m_index.updateFlags(items, addedFlags, removedFlags);
0178     m_index.scheduleCommit();
0179 }
0180
0181 void AkonadiIndexingAgent::itemsRemoved(const Akonadi::Item::List &items)
0182 {
0183     // We optimize and skip the "shouldIndex" call for each item here, since it's
0184     // cheaper to just let Xapian throw an exception for items that were not
0185     // indexed instead of filtering the list here.
0186
0187     m_index.remove(items);
0188     m_index.scheduleCommit();
0189 }
0190
0191 void AkonadiIndexingAgent::itemsMoved(const Akonadi::Item::List &items,
0192                                       const Akonadi::Collection &sourceCollection,
0193                                       const Akonadi::Collection &destinationCollection)
0194 {
0195     const bool indexSource = shouldIndex(sourceCollection);
0196     const bool indexDest = shouldIndex(destinationCollection);
0197
0198     if (indexSource && indexDest) {
0199         m_index.move(items, sourceCollection, destinationCollection);
0200         m_index.scheduleCommit();
0201     } else if (!indexSource && indexDest) {
0202         for (const auto &item : items) {
0203             m_scheduler.addItem(item);
0204         }
0205         m_index.scheduleCommit();
0206     } else if (indexSource && !indexDest) {
0207         m_index.remove(items);
0208         m_index.scheduleCommit();
0209     } else {
0210         // nothing to do
0211     }
0212 }
0213
0214 void AkonadiIndexingAgent::collectionAdded(const Akonadi::Collection &collection, const Akonadi::Collection &parent)
0215 {
0216     Q_UNUSED(parent)
0217
0218     if (!shouldIndex(collection)) {
0219         return;
0220     }
0221
0222     m_index.index(collection);
0223     m_index.scheduleCommit();
0224 }
0225
0226 void AkonadiIndexingAgent::collectionChanged(const Akonadi::Collection &collection, const QSet<QByteArray> &changedAttributes)
0227 {
0228     if (changedAttributes.contains(Akonadi::IndexPolicyAttribute().type())) {
0229         const auto attr = collection.attribute<Akonadi::IndexPolicyAttribute>();
0230         if (attr && !attr->indexingEnabled()) {
0231             // The indexing attribute has changed and is now disabled: remove
0232             // collection and all indexed items
0233             m_index.remove(collection);
0234         } else {
0235             // The indexing attribute has changed and is now missing or enabled,
0236             // schedule full collection sync.
0237             m_scheduler.scheduleCollection(collection, true);
0238         }
0239     }
0240
0241     QSet<QByteArray> changes = changedAttributes;
0242     changes.remove("collectionquota");
0243     changes.remove("timestamp");
0244     changes.remove("imapquota");
0245
0246     if (changes.isEmpty()) {
0247         return;
0248     }
0249
0250     if (changes.contains("ENTITYDISPLAY")) {
0251         // If the name changed we have to reindex all subcollections
0252         auto job = new CollectionUpdateJob(m_index, collection, this);
0253         job->start();
0254     } else {
0255         m_index.index(collection);
0256         m_index.scheduleCommit();
0257     }
0258 }
0259
0260 void AkonadiIndexingAgent::collectionRemoved(const Akonadi::Collection &collection)
0261 {
0262     // We intentionally don't call "shouldIndex" here to make absolutely sure
0263     // that all items are wiped from the index
0264
0265     m_index.remove(collection);
0266     m_index.scheduleCommit();
0267 }
0268
0269 void AkonadiIndexingAgent::collectionMoved(const Akonadi::Collection &collection,
0270                                            const Akonadi::Collection &collectionSource,
0271                                            const Akonadi::Collection &collectionDestination)
0272 {
0273     Q_UNUSED(collectionSource)
0274     Q_UNUSED(collectionDestination)
0275
0276     if (!shouldIndex(collection)) {
0277         return;
0278     }
0279
0280     m_index.remove(collection);
0281     auto job = new CollectionUpdateJob(m_index, collection, this);
0282     job->start();
0283 }
0284
0285 void AkonadiIndexingAgent::cleanup()
0286 {
0287     // Remove all the databases
0288     Akonadi::AgentBase::cleanup();
0289 }
0290
0291 int AkonadiIndexingAgent::numberOfCollectionQueued()
0292 {
0293     return m_scheduler.numberOfCollectionQueued();
0294 }
0295
0296 void AkonadiIndexingAgent::onAbortRequested()
0297 {
0298     KConfigGroup group = config()->group(QStringLiteral("General"));
0299     group.writeEntry("aborted", true);
0300     group.sync();
0301     m_scheduler.abort();
0302 }
0303
0304 void AkonadiIndexingAgent::onOnlineChanged(bool online)
0305 {
0306     // Ignore everything when offline
0307     changeRecorder()->setAllMonitored(online);
0308
0309     // Index items that might have changed while we were offline
0310     if (online) {
0311         // We only reindex if this is not a regular start
0312         KConfigGroup cfg = config()->group(QStringLiteral("General"));
0313         bool aborted = cfg.readEntry("aborted", false);
0314         if (aborted) {
0315             cfg.writeEntry("aborted", false);
0316             cfg.sync();
0317             m_scheduler.scheduleCompleteSync();
0318         }
0319     } else {
0320         // Abort ongoing indexing when switched to offline
0321         onAbortRequested();
0322     }
0323 }
0324
0325 bool AkonadiIndexingAgent::shouldIndex(const Akonadi::Collection &col) const
0326 {
0327     return !col.isVirtual() && (!col.hasAttribute<Akonadi::IndexPolicyAttribute>() || col.attribute<Akonadi::IndexPolicyAttribute>()->indexingEnabled());
0328 }
0329
0330 bool AkonadiIndexingAgent::shouldIndex(const Akonadi::Item &item) const
0331 {
0332     return shouldIndex(item.parentCollection());
0333 }
0334
0335 AKONADI_AGENT_MAIN(AkonadiIndexingAgent)
0336
0337 #include "moc_agent.cpp"