server/handler/itemfetchhelper.cpp

0001
0002 /***************************************************************************
0003  *   SPDX-FileCopyrightText: 2006-2009 Tobias Koenig <tokoe@kde.org>       *
0004  *                                                                         *
0005  *   SPDX-License-Identifier: LGPL-2.0-or-later                            *
0006  ***************************************************************************/
0007
0008 #include "itemfetchhelper.h"
0009
0010 #include "akonadi.h"
0011 #include "connection.h"
0012 #include "handler.h"
0013 #include "handlerhelper.h"
0014 #include "shared/akranges.h"
0015 #include "storage/itemqueryhelper.h"
0016 #include "storage/itemretrievalmanager.h"
0017 #include "storage/itemretrievalrequest.h"
0018 #include "storage/parthelper.h"
0019 #include "storage/parttypehelper.h"
0020 #include "storage/selectquerybuilder.h"
0021 #include "storage/transaction.h"
0022
0023 #include "agentmanagerinterface.h"
0024 #include "akonadiserver_debug.h"
0025 #include "intervalcheck.h"
0026 #include "relationfetchhandler.h"
0027 #include "tagfetchhelper.h"
0028 #include "utils.h"
0029
0030 #include "private/dbus_p.h"
0031
0032 #include <QDateTime>
0033 #include <QSqlQuery>
0034 #include <QStringList>
0035 #include <QVariant>
0036
0037 #include <QElapsedTimer>
0038
0039 using namespace Akonadi;
0040 using namespace Akonadi::Server;
0041 using namespace AkRanges;
0042
0043 #define ENABLE_FETCH_PROFILING 0
0044 #if ENABLE_FETCH_PROFILING
0045 #define BEGIN_TIMER(name)                                                                                                                                      \
0046     QElapsedTimer name##Timer;                                                                                                                                 \
0047     name##Timer.start();
0048
0049 #define END_TIMER(name) const double name##Elapsed = name##Timer.nsecsElapsed() / 1000000.0;
0050 #define PROF_INC(name) ++name;
0051 #else
0052 #define BEGIN_TIMER(name)
0053 #define END_TIMER(name)
0054 #define PROF_INC(name)
0055 #endif
0056
0057 ItemFetchHelper::ItemFetchHelper(Connection *connection,
0058                                  const Scope &scope,
0059                                  const Protocol::ItemFetchScope &itemFetchScope,
0060                                  const Protocol::TagFetchScope &tagFetchScope,
0061                                  AkonadiServer &akonadi,
0062                                  const Protocol::FetchLimit &itemsLimit)
0063     : ItemFetchHelper(connection, connection->context(), scope, itemFetchScope, tagFetchScope, akonadi, itemsLimit)
0064 {
0065 }
0066
0067 ItemFetchHelper::ItemFetchHelper(Connection *connection,
0068                                  const CommandContext &context,
0069                                  const Scope &scope,
0070                                  const Protocol::ItemFetchScope &itemFetchScope,
0071                                  const Protocol::TagFetchScope &tagFetchScope,
0072                                  AkonadiServer &akonadi,
0073                                  const Protocol::FetchLimit &itemsLimit)
0074     : mConnection(connection)
0075     , mContext(context)
0076     , mScope(scope)
0077     , mItemFetchScope(itemFetchScope)
0078     , mTagFetchScope(tagFetchScope)
0079     , mAkonadi(akonadi)
0080     , mItemsLimit(itemsLimit)
0081     , mItemQuery(PimItem::tableName())
0082     , mPimItemQueryAlias(QLatin1StringView("pimItem_alias"))
0083 {
0084     std::fill(mItemQueryColumnMap, mItemQueryColumnMap + ItemQueryColumnCount, -1);
0085 }
0086
0087 void ItemFetchHelper::disableATimeUpdates()
0088 {
0089     mUpdateATimeEnabled = false;
0090 }
0091
0092 enum PartQueryColumns {
0093     PartQueryPimIdColumn,
0094     PartQueryTypeIdColumn,
0095     PartQueryDataColumn,
0096     PartQueryStorageColumn,
0097     PartQueryVersionColumn,
0098     PartQueryDataSizeColumn
0099 };
0100
0101 QSqlQuery ItemFetchHelper::buildPartQuery(const QList<QByteArray> &partList, bool allPayload, bool allAttrs)
0102 {
0103     /// TODO: merge with ItemQuery
0104     QueryBuilder partQuery(PimItem::tableName());
0105     if (mItemsLimit.limit() > 0) {
0106         partQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
0107     }
0108
0109     if (!partList.isEmpty() || allPayload || allAttrs) {
0110         partQuery.addJoin(QueryBuilder::InnerJoin, Part::tableName(), partQuery.getTableWithColumn(PimItem::idColumn()), Part::pimItemIdFullColumnName());
0111         partQuery.addColumn(partQuery.getTableWithColumn(PimItem::idColumn()));
0112         partQuery.addColumn(Part::partTypeIdFullColumnName());
0113         partQuery.addColumn(Part::dataFullColumnName());
0114         partQuery.addColumn(Part::storageFullColumnName());
0115         partQuery.addColumn(Part::versionFullColumnName());
0116         partQuery.addColumn(Part::datasizeFullColumnName());
0117
0118         partQuery.addSortColumn(partQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
0119
0120         if (!partList.isEmpty() || allPayload || allAttrs) {
0121             Query::Condition cond(Query::Or);
0122             for (const QByteArray &b : std::as_const(partList)) {
0123                 if (b.startsWith("PLD") || b.startsWith("ATR")) {
0124                     cond.addValueCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartTypeHelper::fromFqName(b).id());
0125                 }
0126             }
0127             if (allPayload || allAttrs) {
0128                 partQuery.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName());
0129                 if (allPayload) {
0130                     cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
0131                 }
0132                 if (allAttrs) {
0133                     cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("ATR"));
0134                 }
0135             }
0136
0137             partQuery.addCondition(cond);
0138         }
0139
0140         ItemQueryHelper::scopeToQuery(mScope, mContext, partQuery);
0141
0142         if (!partQuery.exec()) {
0143             throw HandlerException("Unable to list item parts");
0144         }
0145         partQuery.query().next();
0146     }
0147
0148     return partQuery.query();
0149 }
0150
0151 QSqlQuery ItemFetchHelper::buildItemQuery()
0152 {
0153     int column = 0;
0154 #define ADD_COLUMN(colName, colId)                                                                                                                             \
0155     {                                                                                                                                                          \
0156         mItemQuery.addColumn(colName);                                                                                                                         \
0157         mItemQueryColumnMap[colId] = column++;                                                                                                                 \
0158     }
0159     ADD_COLUMN(PimItem::idFullColumnName(), ItemQueryPimItemIdColumn);
0160     if (mItemFetchScope.fetchRemoteId()) {
0161         ADD_COLUMN(PimItem::remoteIdFullColumnName(), ItemQueryPimItemRidColumn)
0162     }
0163     ADD_COLUMN(PimItem::mimeTypeIdFullColumnName(), ItemQueryMimeTypeIdColumn)
0164     ADD_COLUMN(PimItem::revFullColumnName(), ItemQueryRevColumn)
0165     if (mItemFetchScope.fetchRemoteRevision()) {
0166         ADD_COLUMN(PimItem::remoteRevisionFullColumnName(), ItemQueryRemoteRevisionColumn)
0167     }
0168     if (mItemFetchScope.fetchSize()) {
0169         ADD_COLUMN(PimItem::sizeFullColumnName(), ItemQuerySizeColumn)
0170     }
0171     if (mItemFetchScope.fetchMTime()) {
0172         ADD_COLUMN(PimItem::datetimeFullColumnName(), ItemQueryDatetimeColumn)
0173     }
0174     ADD_COLUMN(PimItem::collectionIdFullColumnName(), ItemQueryCollectionIdColumn)
0175     if (mItemFetchScope.fetchGID()) {
0176         ADD_COLUMN(PimItem::gidFullColumnName(), ItemQueryPimItemGidColumn)
0177     }
0178 #undef ADD_COLUMN
0179
0180     mItemQuery.addSortColumn(PimItem::idFullColumnName(), static_cast<Query::SortOrder>(mItemsLimit.sortOrder()));
0181     if (mItemsLimit.limit() > 0) {
0182         mItemQuery.setLimit(mItemsLimit.limit(), mItemsLimit.limitOffset());
0183     }
0184
0185     ItemQueryHelper::scopeToQuery(mScope, mContext, mItemQuery);
0186
0187     if (mItemFetchScope.changedSince().isValid()) {
0188         mItemQuery.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mItemFetchScope.changedSince().toUTC());
0189     }
0190
0191     if (!mItemQuery.exec()) {
0192         throw HandlerException("Unable to list items");
0193     }
0194
0195     mItemQuery.query().next();
0196
0197     return mItemQuery.query();
0198 }
0199
0200 enum FlagQueryColumns {
0201     FlagQueryPimItemIdColumn,
0202     FlagQueryFlagIdColumn,
0203 };
0204
0205 QSqlQuery ItemFetchHelper::buildFlagQuery()
0206 {
0207     QueryBuilder flagQuery(PimItem::tableName());
0208     if (mItemsLimit.limit() > 0) {
0209         flagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
0210     }
0211
0212     flagQuery.addJoin(QueryBuilder::InnerJoin,
0213                       PimItemFlagRelation::tableName(),
0214                       flagQuery.getTableWithColumn(PimItem::idColumn()),
0215                       PimItemFlagRelation::leftFullColumnName());
0216
0217     flagQuery.addColumn(flagQuery.getTableWithColumn(PimItem::idColumn()));
0218     flagQuery.addColumn(PimItemFlagRelation::rightFullColumnName());
0219
0220     ItemQueryHelper::scopeToQuery(mScope, mContext, flagQuery);
0221     flagQuery.addSortColumn(flagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
0222
0223     if (!flagQuery.exec()) {
0224         throw HandlerException("Unable to retrieve item flags");
0225     }
0226
0227     flagQuery.query().next();
0228
0229     return flagQuery.query();
0230 }
0231
0232 enum TagQueryColumns {
0233     TagQueryItemIdColumn,
0234     TagQueryTagIdColumn,
0235 };
0236
0237 QSqlQuery ItemFetchHelper::buildTagQuery()
0238 {
0239     QueryBuilder tagQuery(PimItem::tableName());
0240     if (mItemsLimit.limit() > 0) {
0241         tagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
0242     }
0243
0244     tagQuery.addJoin(QueryBuilder::InnerJoin,
0245                      PimItemTagRelation::tableName(),
0246                      tagQuery.getTableWithColumn(PimItem::idColumn()),
0247                      PimItemTagRelation::leftFullColumnName());
0248     tagQuery.addJoin(QueryBuilder::InnerJoin, Tag::tableName(), Tag::idFullColumnName(), PimItemTagRelation::rightFullColumnName());
0249     tagQuery.addColumn(tagQuery.getTableWithColumn(PimItem::idColumn()));
0250     tagQuery.addColumn(Tag::idFullColumnName());
0251
0252     ItemQueryHelper::scopeToQuery(mScope, mContext, tagQuery);
0253     tagQuery.addSortColumn(tagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
0254
0255     if (!tagQuery.exec()) {
0256         throw HandlerException("Unable to retrieve item tags");
0257     }
0258
0259     tagQuery.query().next();
0260
0261     return tagQuery.query();
0262 }
0263
0264 enum VRefQueryColumns {
0265     VRefQueryCollectionIdColumn,
0266     VRefQueryItemIdColumn,
0267 };
0268
0269 QSqlQuery ItemFetchHelper::buildVRefQuery()
0270 {
0271     QueryBuilder vRefQuery(PimItem::tableName());
0272     if (mItemsLimit.limit() > 0) {
0273         vRefQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
0274     }
0275
0276     vRefQuery.addJoin(QueryBuilder::LeftJoin,
0277                       CollectionPimItemRelation::tableName(),
0278                       CollectionPimItemRelation::rightFullColumnName(),
0279                       vRefQuery.getTableWithColumn(PimItem::idColumn()));
0280     vRefQuery.addColumn(CollectionPimItemRelation::leftFullColumnName());
0281     vRefQuery.addColumn(CollectionPimItemRelation::rightFullColumnName());
0282     ItemQueryHelper::scopeToQuery(mScope, mContext, vRefQuery);
0283     vRefQuery.addSortColumn(vRefQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
0284
0285     if (!vRefQuery.exec()) {
0286         throw HandlerException("Unable to retrieve virtual references");
0287     }
0288
0289     vRefQuery.query().next();
0290
0291     return vRefQuery.query();
0292 }
0293
0294 bool ItemFetchHelper::isScopeLocal(const Scope &scope)
0295 {
0296     // The only agent allowed to override local scope is the Baloo Indexer
0297     if (!mConnection->sessionId().startsWith("akonadi_indexing_agent")) {
0298         return false;
0299     }
0300
0301     // Get list of all resources that own all items in the scope
0302     QueryBuilder qb(PimItem::tableName(), QueryBuilder::Select);
0303     qb.setDistinct(true);
0304     qb.addColumn(Resource::nameFullColumnName());
0305     qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
0306     qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
0307     ItemQueryHelper::scopeToQuery(scope, mContext, qb);
0308     if (mContext.resource().isValid()) {
0309         qb.addValueCondition(Resource::nameFullColumnName(), Query::NotEquals, mContext.resource().name());
0310     }
0311
0312     if (!qb.exec()) {
0313         throw HandlerException("Failed to query database");
0314         return false;
0315     }
0316
0317     // If there is more than one resource, i.e. this is a fetch from multiple
0318     // collections, then don't bother and just return FALSE. This case is aimed
0319     // specifically on Baloo, which fetches items from each collection independently,
0320     // so it will pass this check.
0321     QSqlQuery query = qb.query();
0322     if (query.size() != 1) {
0323         return false;
0324     }
0325
0326     query.next();
0327     const QString resourceName = query.value(0).toString();
0328     query.finish();
0329
0330     org::freedesktop::Akonadi::AgentManager manager(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus());
0331     const QString typeIdentifier = manager.agentInstanceType(resourceName);
0332     const QVariantMap properties = manager.agentCustomProperties(typeIdentifier);
0333     return properties.value(QStringLiteral("HasLocalStorage"), false).toBool();
0334 }
0335
0336 DataStore *ItemFetchHelper::storageBackend() const
0337 {
0338     if (mConnection) {
0339         if (auto store = mConnection->storageBackend()) {
0340             return store;
0341         }
0342     }
0343
0344     return DataStore::self();
0345 }
0346
0347 bool ItemFetchHelper::fetchItems(std::function<void(Protocol::FetchItemsResponse &&)> &&itemCallback)
0348 {
0349     BEGIN_TIMER(fetch)
0350
0351     // retrieve missing parts
0352     // HACK: isScopeLocal() is a workaround for resources that have cache expiration
0353     // because when the cache expires, Baloo is not able to content of the items. So
0354     // we allow fetch of items that belong to local resources (like maildir) to ignore
0355     // cacheOnly and retrieve missing parts from the resource. However ItemRetriever
0356     // is painfully slow with many items and is generally designed to fetch a few
0357     // messages, not all of them. In the long term, we need a better way to do this.
0358     BEGIN_TIMER(itemRetriever)
0359     BEGIN_TIMER(scopeLocal)
0360 #if ENABLE_FETCH_PROFILING
0361     double scopeLocalElapsed = 0;
0362 #endif
0363     if (!mItemFetchScope.cacheOnly() || isScopeLocal(mScope)) {
0364 #if ENABLE_FETCH_PROFILING
0365         scopeLocalElapsed = scopeLocalTimer.elapsed();
0366 #endif
0367
0368         // trigger a collection sync if configured to do so
0369         triggerOnDemandFetch();
0370
0371         // Prepare for a call to ItemRetriever::exec();
0372         // From a resource perspective the only parts that can be fetched are payloads.
0373         ItemRetriever retriever(mAkonadi.itemRetrievalManager(), mConnection, mContext);
0374         retriever.setScope(mScope);
0375         retriever.setRetrieveParts(mItemFetchScope.requestedPayloads());
0376         retriever.setRetrieveFullPayload(mItemFetchScope.fullPayload());
0377         retriever.setChangedSince(mItemFetchScope.changedSince());
0378         if (!retriever.exec() && !mItemFetchScope.ignoreErrors()) { // There we go, retrieve the missing parts from the resource.
0379             if (mContext.resource().isValid()) {
0380                 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1, resource %2) : %3")
0381                                            .arg(mContext.collectionId())
0382                                            .arg(mContext.resource().id())
0383                                            .arg(QString::fromLatin1(retriever.lastError())));
0384             } else {
0385                 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1) : %2")
0386                                            .arg(mContext.collectionId())
0387                                            .arg(QString::fromLatin1(retriever.lastError())));
0388             }
0389         }
0390     }
0391     END_TIMER(itemRetriever)
0392
0393     BEGIN_TIMER(items)
0394     QSqlQuery itemQuery = buildItemQuery();
0395     END_TIMER(items)
0396
0397     // error if query did not find any item and scope is not listing items but
0398     // a request for a specific item
0399     if (!itemQuery.isValid()) {
0400         if (mItemFetchScope.ignoreErrors()) {
0401             return true;
0402         }
0403         switch (mScope.scope()) {
0404         case Scope::Uid: // fall through
0405         case Scope::Rid: // fall through
0406         case Scope::HierarchicalRid: // fall through
0407         case Scope::Gid:
0408             throw HandlerException("Item query returned empty result set");
0409             break;
0410         default:
0411             break;
0412         }
0413     }
0414     // build part query if needed
0415     BEGIN_TIMER(parts)
0416     QSqlQuery partQuery(storageBackend()->database());
0417     if (!mItemFetchScope.requestedParts().isEmpty() || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
0418         partQuery = buildPartQuery(mItemFetchScope.requestedParts(), mItemFetchScope.fullPayload(), mItemFetchScope.allAttributes());
0419     }
0420     END_TIMER(parts)
0421
0422     // build flag query if needed
0423     BEGIN_TIMER(flags)
0424     QSqlQuery flagQuery(storageBackend()->database());
0425     if (mItemFetchScope.fetchFlags()) {
0426         flagQuery = buildFlagQuery();
0427     }
0428     END_TIMER(flags)
0429
0430     // build tag query if needed
0431     BEGIN_TIMER(tags)
0432     QSqlQuery tagQuery(storageBackend()->database());
0433     if (mItemFetchScope.fetchTags()) {
0434         tagQuery = buildTagQuery();
0435     }
0436     END_TIMER(tags)
0437
0438     BEGIN_TIMER(vRefs)
0439     QSqlQuery vRefQuery(storageBackend()->database());
0440     if (mItemFetchScope.fetchVirtualReferences()) {
0441         vRefQuery = buildVRefQuery();
0442     }
0443     END_TIMER(vRefs)
0444
0445 #if ENABLE_FETCH_PROFILING
0446     int itemsCount = 0;
0447     int flagsCount = 0;
0448     int partsCount = 0;
0449     int tagsCount = 0;
0450     int vRefsCount = 0;
0451 #endif
0452
0453     BEGIN_TIMER(processing)
0454     QHash<qint64, QByteArray> flagIdNameCache;
0455     QHash<qint64, QString> mimeTypeIdNameCache;
0456     QHash<qint64, QByteArray> partTypeIdNameCache;
0457     while (itemQuery.isValid()) {
0458         PROF_INC(itemsCount)
0459
0460         const qint64 pimItemId = extractQueryResult(itemQuery, ItemQueryPimItemIdColumn).toLongLong();
0461         const int pimItemRev = extractQueryResult(itemQuery, ItemQueryRevColumn).toInt();
0462
0463         Protocol::FetchItemsResponse response;
0464         response.setId(pimItemId);
0465         response.setRevision(pimItemRev);
0466         const qint64 mimeTypeId = extractQueryResult(itemQuery, ItemQueryMimeTypeIdColumn).toLongLong();
0467         auto mtIter = mimeTypeIdNameCache.find(mimeTypeId);
0468         if (mtIter == mimeTypeIdNameCache.end()) {
0469             mtIter = mimeTypeIdNameCache.insert(mimeTypeId, MimeType::retrieveById(mimeTypeId).name());
0470         }
0471         response.setMimeType(mtIter.value());
0472         if (mItemFetchScope.fetchRemoteId()) {
0473             response.setRemoteId(extractQueryResult(itemQuery, ItemQueryPimItemRidColumn).toString());
0474         }
0475         response.setParentId(extractQueryResult(itemQuery, ItemQueryCollectionIdColumn).toLongLong());
0476
0477         if (mItemFetchScope.fetchSize()) {
0478             response.setSize(extractQueryResult(itemQuery, ItemQuerySizeColumn).toLongLong());
0479         }
0480         if (mItemFetchScope.fetchMTime()) {
0481             response.setMTime(Utils::variantToDateTime(extractQueryResult(itemQuery, ItemQueryDatetimeColumn)));
0482         }
0483         if (mItemFetchScope.fetchRemoteRevision()) {
0484             response.setRemoteRevision(extractQueryResult(itemQuery, ItemQueryRemoteRevisionColumn).toString());
0485         }
0486         if (mItemFetchScope.fetchGID()) {
0487             response.setGid(extractQueryResult(itemQuery, ItemQueryPimItemGidColumn).toString());
0488         }
0489
0490         if (mItemFetchScope.fetchFlags()) {
0491             QList<QByteArray> flags;
0492             while (flagQuery.isValid()) {
0493                 const qint64 id = flagQuery.value(FlagQueryPimItemIdColumn).toLongLong();
0494                 if (id > pimItemId) {
0495                     flagQuery.next();
0496                     continue;
0497                 } else if (id < pimItemId) {
0498                     break;
0499                 }
0500                 const qint64 flagId = flagQuery.value(FlagQueryFlagIdColumn).toLongLong();
0501                 auto flagNameIter = flagIdNameCache.find(flagId);
0502                 if (flagNameIter == flagIdNameCache.end()) {
0503                     flagNameIter = flagIdNameCache.insert(flagId, Flag::retrieveById(flagId).name().toUtf8());
0504                 }
0505                 flags << flagNameIter.value();
0506                 flagQuery.next();
0507             }
0508             response.setFlags(flags);
0509         }
0510
0511         if (mItemFetchScope.fetchTags()) {
0512             QList<qint64> tagIds;
0513             QList<Protocol::FetchTagsResponse> tags;
0514             while (tagQuery.isValid()) {
0515                 PROF_INC(tagsCount)
0516                 const qint64 id = tagQuery.value(TagQueryItemIdColumn).toLongLong();
0517                 if (id > pimItemId) {
0518                     tagQuery.next();
0519                     continue;
0520                 } else if (id < pimItemId) {
0521                     break;
0522                 }
0523                 tagIds << tagQuery.value(TagQueryTagIdColumn).toLongLong();
0524                 tagQuery.next();
0525             }
0526
0527             if (mTagFetchScope.fetchIdOnly()) {
0528                 tags = tagIds | Views::transform([](const auto tagId) {
0529                            Protocol::FetchTagsResponse resp;
0530                            resp.setId(tagId);
0531                            return resp;
0532                        })
0533                     | Actions::toQVector;
0534             } else {
0535                 tags = tagIds | Views::transform([this](const auto tagId) {
0536                            return HandlerHelper::fetchTagsResponse(Tag::retrieveById(tagId), mTagFetchScope, mConnection);
0537                        })
0538                     | Actions::toQVector;
0539             }
0540             response.setTags(tags);
0541         }
0542
0543         if (mItemFetchScope.fetchVirtualReferences()) {
0544             QList<qint64> vRefs;
0545             while (vRefQuery.isValid()) {
0546                 PROF_INC(vRefsCount)
0547                 const qint64 id = vRefQuery.value(VRefQueryItemIdColumn).toLongLong();
0548                 if (id > pimItemId) {
0549                     vRefQuery.next();
0550                     continue;
0551                 } else if (id < pimItemId) {
0552                     break;
0553                 }
0554                 vRefs << vRefQuery.value(VRefQueryCollectionIdColumn).toLongLong();
0555                 vRefQuery.next();
0556             }
0557             response.setVirtualReferences(vRefs);
0558         }
0559
0560         if (mItemFetchScope.fetchRelations()) {
0561             SelectQueryBuilder<Relation> qb;
0562             Query::Condition condition;
0563             condition.setSubQueryMode(Query::Or);
0564             condition.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, pimItemId);
0565             condition.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, pimItemId);
0566             qb.addCondition(condition);
0567             qb.addGroupColumns(QStringList() << Relation::leftIdColumn() << Relation::rightIdColumn() << Relation::typeIdColumn()
0568                                              << Relation::remoteIdColumn());
0569             if (!qb.exec()) {
0570                 throw HandlerException("Unable to list item relations");
0571             }
0572             QList<Protocol::FetchRelationsResponse> relations;
0573             const auto result = qb.result();
0574             relations.reserve(result.size());
0575             for (const Relation &rel : result) {
0576                 relations.push_back(HandlerHelper::fetchRelationsResponse(rel));
0577                 ;
0578             }
0579             response.setRelations(relations);
0580         }
0581
0582         if (mItemFetchScope.ancestorDepth() != Protocol::ItemFetchScope::NoAncestor) {
0583             response.setAncestors(ancestorsForItem(response.parentId()));
0584         }
0585
0586         bool skipItem = false;
0587
0588         QList<QByteArray> cachedParts;
0589         QList<Protocol::StreamPayloadResponse> parts;
0590         while (partQuery.isValid()) {
0591             PROF_INC(partsCount)
0592             const qint64 id = partQuery.value(PartQueryPimIdColumn).toLongLong();
0593             if (id > pimItemId) {
0594                 partQuery.next();
0595                 continue;
0596             } else if (id < pimItemId) {
0597                 break;
0598             }
0599
0600             const qint64 partTypeId = partQuery.value(PartQueryTypeIdColumn).toLongLong();
0601             auto ptIter = partTypeIdNameCache.find(partTypeId);
0602             if (ptIter == partTypeIdNameCache.end()) {
0603                 ptIter = partTypeIdNameCache.insert(partTypeId, PartTypeHelper::fullName(PartType::retrieveById(partTypeId)).toUtf8());
0604             }
0605             Protocol::PartMetaData metaPart;
0606             Protocol::StreamPayloadResponse partData;
0607             partData.setPayloadName(ptIter.value());
0608             metaPart.setName(ptIter.value());
0609             metaPart.setVersion(partQuery.value(PartQueryVersionColumn).toInt());
0610             metaPart.setSize(partQuery.value(PartQueryDataSizeColumn).toLongLong());
0611
0612             const QByteArray data = Utils::variantToByteArray(partQuery.value(PartQueryDataColumn));
0613             if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
0614                 if (!data.isEmpty()) {
0615                     cachedParts << ptIter.value();
0616                 }
0617                 partQuery.next();
0618             } else {
0619                 if (mItemFetchScope.ignoreErrors() && data.isEmpty()) {
0620                     // We wanted the payload, couldn't get it, and are ignoring errors. Skip the item.
0621                     // This is not an error though, it's fine to have empty payload parts (to denote existing but not cached parts)
0622                     qCDebug(AKONADISERVER_LOG) << "item" << id << "has an empty payload part in parttable for part" << metaPart.name();
0623                     skipItem = true;
0624                     break;
0625                 }
0626                 metaPart.setStorageType(static_cast<Protocol::PartMetaData::StorageType>(partQuery.value(PartQueryStorageColumn).toInt()));
0627                 if (data.isEmpty()) {
0628                     partData.setData(QByteArray(""));
0629                 } else {
0630                     partData.setData(data);
0631                 }
0632                 partData.setMetaData(metaPart);
0633
0634                 if (mItemFetchScope.requestedParts().contains(ptIter.value()) || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
0635                     parts.append(partData);
0636                 }
0637
0638                 partQuery.next();
0639             }
0640         }
0641         response.setParts(parts);
0642
0643         if (skipItem) {
0644             itemQuery.next();
0645             continue;
0646         }
0647
0648         if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
0649             response.setCachedParts(cachedParts);
0650         }
0651
0652         if (itemCallback) {
0653             itemCallback(std::move(response));
0654         } else {
0655             mConnection->sendResponse(std::move(response));
0656         }
0657
0658         itemQuery.next();
0659     }
0660     tagQuery.finish();
0661     flagQuery.finish();
0662     partQuery.finish();
0663     vRefQuery.finish();
0664     itemQuery.finish();
0665     END_TIMER(processing)
0666
0667     // update atime (only if the payload was actually requested, otherwise a simple resource sync prevents cache clearing)
0668     BEGIN_TIMER(aTime)
0669     if (mUpdateATimeEnabled && (needsAccessTimeUpdate(mItemFetchScope.requestedParts()) || mItemFetchScope.fullPayload())) {
0670         updateItemAccessTime();
0671     }
0672     END_TIMER(aTime)
0673
0674     END_TIMER(fetch)
0675 #if ENABLE_FETCH_PROFILING
0676     qCDebug(AKONADISERVER_LOG) << "ItemFetchHelper execution stats:";
0677     qCDebug(AKONADISERVER_LOG) << "\tItems query:" << itemsElapsed << "ms," << itemsCount << " items in total";
0678     qCDebug(AKONADISERVER_LOG) << "\tFlags query:" << flagsElapsed << "ms, " << flagsCount << " flags in total";
0679     qCDebug(AKONADISERVER_LOG) << "\tParts query:" << partsElapsed << "ms, " << partsCount << " parts in total";
0680     qCDebug(AKONADISERVER_LOG) << "\tTags query: " << tagsElapsed << "ms, " << tagsCount << " tags in total";
0681     qCDebug(AKONADISERVER_LOG) << "\tVRefs query:" << vRefsElapsed << "ms, " << vRefsCount << " vRefs in total";
0682     qCDebug(AKONADISERVER_LOG) << "\t------------";
0683     qCDebug(AKONADISERVER_LOG) << "\tItem retriever:" << itemRetrieverElapsed << "ms (scope local:" << scopeLocalElapsed << "ms)";
0684     qCDebug(AKONADISERVER_LOG) << "\tTotal query:" << (itemsElapsed + flagsElapsed + partsElapsed + tagsElapsed + vRefsElapsed) << "ms";
0685     qCDebug(AKONADISERVER_LOG) << "\tTotal processing: " << processingElapsed << "ms";
0686     qCDebug(AKONADISERVER_LOG) << "\tATime update:" << aTimeElapsed << "ms";
0687     qCDebug(AKONADISERVER_LOG) << "\t============";
0688     qCDebug(AKONADISERVER_LOG) << "\tTotal FETCH:" << fetchElapsed << "ms";
0689     qCDebug(AKONADISERVER_LOG);
0690     qCDebug(AKONADISERVER_LOG);
0691 #endif
0692
0693     return true;
0694 }
0695
0696 bool ItemFetchHelper::needsAccessTimeUpdate(const QList<QByteArray> &parts)
0697 {
0698     // TODO technically we should compare the part list with the cache policy of
0699     // the parent collection of the retrieved items, but that's kinda expensive
0700     // Only updating the atime if the full payload was requested is a good
0701     // approximation though.
0702     return parts.contains(AKONADI_PARAM_PLD_RFC822);
0703 }
0704
0705 void ItemFetchHelper::updateItemAccessTime()
0706 {
0707     Transaction transaction(storageBackend(), QStringLiteral("update atime"));
0708     QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
0709     qb.setColumnValue(PimItem::atimeColumn(), QDateTime::currentDateTimeUtc());
0710     ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
0711
0712     if (!qb.exec()) {
0713         qCWarning(AKONADISERVER_LOG) << "Unable to update item access time";
0714     } else {
0715         transaction.commit();
0716     }
0717 }
0718
0719 void ItemFetchHelper::triggerOnDemandFetch()
0720 {
0721     if (mContext.collectionId() <= 0 || mItemFetchScope.cacheOnly()) {
0722         return;
0723     }
0724
0725     Collection collection = mContext.collection();
0726
0727     // HACK: don't trigger on-demand syncing if the resource is the one triggering it
0728     if (mConnection->sessionId() == collection.resource().name().toLatin1()) {
0729         return;
0730     }
0731
0732     storageBackend()->activeCachePolicy(collection);
0733     if (!collection.cachePolicySyncOnDemand()) {
0734         return;
0735     }
0736
0737     mConnection->akonadi().intervalChecker().requestCollectionSync(collection);
0738 }
0739
0740 QList<Protocol::Ancestor> ItemFetchHelper::ancestorsForItem(Collection::Id parentColId)
0741 {
0742     if (mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::NoAncestor || parentColId == 0) {
0743         return QList<Protocol::Ancestor>();
0744     }
0745     const auto it = mAncestorCache.constFind(parentColId);
0746     if (it != mAncestorCache.cend()) {
0747         return *it;
0748     }
0749
0750     QList<Protocol::Ancestor> ancestors;
0751     Collection col = Collection::retrieveById(parentColId);
0752     const int depthNum = mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::ParentAncestor ? 1 : INT_MAX;
0753     for (int i = 0; i < depthNum; ++i) {
0754         if (!col.isValid()) {
0755             Protocol::Ancestor ancestor;
0756             ancestor.setId(0);
0757             ancestors << ancestor;
0758             break;
0759         }
0760         Protocol::Ancestor ancestor;
0761         ancestor.setId(col.id());
0762         ancestor.setRemoteId(col.remoteId());
0763         ancestors << ancestor;
0764         col = col.parent();
0765     }
0766     mAncestorCache.insert(parentColId, ancestors);
0767     return ancestors;
0768 }
0769
0770 QVariant ItemFetchHelper::extractQueryResult(const QSqlQuery &query, ItemFetchHelper::ItemQueryColumns column) const
0771 {
0772     const int colId = mItemQueryColumnMap[column];
0773     Q_ASSERT(colId >= 0);
0774     return query.value(colId);
0775 }