File indexing completed on 2024-05-26 05:27:50
0001 /* 0002 * Copyright (C) 2018 Christian Mollekopf <chrigi_1@fastmail.fm> 0003 * 0004 * This program is free software; you can redistribute it and/or modify 0005 * it under the terms of the GNU General Public License as published by 0006 * the Free Software Foundation; either version 2 of the License, or 0007 * (at your option) any later version. 0008 * 0009 * This program is distributed in the hope that it will be useful, 0010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 0011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 0012 * GNU General Public License for more details. 0013 * 0014 * You should have received a copy of the GNU General Public License 0015 * along with this program; if not, write to the 0016 * Free Software Foundation, Inc., 0017 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 0018 */ 0019 0020 #include "webdav.h" 0021 0022 #include "applicationdomaintype.h" 0023 #include "resourceconfig.h" 0024 0025 #include <KDAV2/DavCollectionDeleteJob> 0026 #include <KDAV2/DavCollectionModifyJob> 0027 #include <KDAV2/DavCollectionCreateJob> 0028 #include <KDAV2/DavCollectionsFetchJob> 0029 #include <KDAV2/DavDiscoveryJob> 0030 #include <KDAV2/DavItemCreateJob> 0031 #include <KDAV2/DavItemDeleteJob> 0032 #include <KDAV2/DavItemsFetchJob> 0033 #include <KDAV2/DavItemFetchJob> 0034 #include <KDAV2/DavItemModifyJob> 0035 #include <KDAV2/DavItemsListJob> 0036 #include <KDAV2/DavPrincipalHomesetsFetchJob> 0037 0038 #include <QNetworkReply> 0039 #include <QColor> 0040 0041 static int translateDavError(KJob *job) 0042 { 0043 using Sink::ApplicationDomain::ErrorCode; 0044 0045 const int responseCode = static_cast<KDAV2::DavJobBase *>(job)->latestResponseCode(); 0046 SinkWarning() << "Response code: " << responseCode; 0047 0048 switch (responseCode) { 0049 case QNetworkReply::HostNotFoundError: 0050 case QNetworkReply::ContentNotFoundError: //If we can't find the content we probably messed up the url configuration 0051 case QNetworkReply::UnknownNetworkError: //That's what I got for a create job without any network at all 0052 return ErrorCode::NoServerError; 0053 case QNetworkReply::AuthenticationRequiredError: 0054 case QNetworkReply::InternalServerError: //The kolab server reports a HTTP 500 instead of 401 on invalid credentials (we could introspect the error message for the 401 error code) 0055 case QNetworkReply::OperationCanceledError: // Since we don't login we will just not have the necessary permissions ot view the object 0056 return ErrorCode::LoginError; 0057 case QNetworkReply::ContentConflictError: 0058 case QNetworkReply::UnknownContentError: 0059 return ErrorCode::SynchronizationConflictError; 0060 } 0061 return ErrorCode::UnknownError; 0062 } 0063 0064 static KAsync::Job<void> runJob(KJob *job) 0065 { 0066 return KAsync::start<void>([job](KAsync::Future<void> &future) { 0067 QObject::connect(job, &KJob::result, [&future](KJob *job) { 0068 SinkTrace() << "Job done: " << job->metaObject()->className(); 0069 if (job->error()) { 0070 SinkWarning() << "Job failed: " << job->errorString() << job->metaObject()->className() << job->error(); 0071 auto proxyError = translateDavError(job); 0072 future.setError(proxyError, job->errorString()); 0073 } else { 0074 future.setFinished(); 0075 } 0076 }); 0077 SinkTrace() << "Starting job: " << job->metaObject()->className(); 0078 job->start(); 0079 }); 0080 } 0081 0082 template <typename T> 0083 static KAsync::Job<T> runJob(KJob *job, const std::function<T(KJob *)> &func) 0084 { 0085 return KAsync::start<T>([job, func](KAsync::Future<T> &future) { 0086 QObject::connect(job, &KJob::result, [&future, func](KJob *job) { 0087 SinkTrace() << "Job done: " << job->metaObject()->className(); 0088 if (job->error()) { 0089 SinkWarning() << "Job failed: " << job->errorString() << job->metaObject()->className() << job->error(); 0090 auto proxyError = translateDavError(job); 0091 future.setError(proxyError, job->errorString()); 0092 } else { 0093 future.setValue(func(job)); 0094 future.setFinished(); 0095 } 0096 }); 0097 SinkTrace() << "Starting job: " << job->metaObject()->className(); 0098 job->start(); 0099 }); 0100 } 0101 0102 WebDavSynchronizer::WebDavSynchronizer(const Sink::ResourceContext &context, KDAV2::Protocol protocol, const QByteArray &collectionType, const QByteArrayList &entityTypes) 0103 : Sink::Synchronizer(context), 0104 mProtocol(protocol), 0105 mCollectionType(collectionType), 0106 mEntityTypes(entityTypes) 0107 { 0108 auto config = ResourceConfig::getConfiguration(context.instanceId()); 0109 0110 mServer = QUrl::fromUserInput(config.value("server").toString()); 0111 mUsername = config.value("username").toString(); 0112 } 0113 0114 QList<Sink::Synchronizer::SyncRequest> WebDavSynchronizer::getSyncRequests(const Sink::QueryBase &query) 0115 { 0116 QList<Synchronizer::SyncRequest> list; 0117 if (!query.type().isEmpty()) { 0118 // We want to synchronize something specific 0119 list << Synchronizer::SyncRequest{query}; 0120 } else { 0121 // We want to synchronize everything 0122 list << Synchronizer::SyncRequest{Sink::QueryBase(mCollectionType)}; 0123 //This request depends on the previous one so we flush first. 0124 for (const auto &type : mEntityTypes) { 0125 //TODO one flush would be enough 0126 list << Synchronizer::SyncRequest{Sink::QueryBase{type}, QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; 0127 } 0128 } 0129 return list; 0130 } 0131 0132 KAsync::Job<void> WebDavSynchronizer::synchronizeWithSource(const Sink::QueryBase &query) 0133 { 0134 return discoverServer().then([this, query] (const KDAV2::DavUrl &serverUrl) { 0135 SinkLogCtx(mLogCtx) << "Synchronizing" << query.type() << "through WebDAV at:" << serverUrl.url(); 0136 if (query.type() == mCollectionType) { 0137 return runJob<KDAV2::DavCollection::List>(new KDAV2::DavCollectionsFetchJob{ serverUrl }, 0138 [](KJob *job) { return static_cast<KDAV2::DavCollectionsFetchJob *>(job)->collections(); }) 0139 .then([this](const KDAV2::DavCollection::List &collections) { 0140 0141 QSet<QByteArray> collectionRemoteIDs; 0142 for (const auto &collection : collections) { 0143 collectionRemoteIDs.insert(resourceID(collection)); 0144 } 0145 int count = scanForRemovals(mCollectionType, [&](const QByteArray &remoteId) { 0146 return collectionRemoteIDs.contains(remoteId); 0147 }); 0148 SinkLogCtx(mLogCtx) << "Removed " << count << " collections"; 0149 updateLocalCollections(collections); 0150 }); 0151 } else if (mEntityTypes.contains(query.type())) { 0152 const QSet<QByteArray> collectionsToSync = [&] { 0153 if (query.hasFilter(mCollectionType)) { 0154 auto folderFilter = query.getFilter(mCollectionType); 0155 auto localIds = resolveFilter(folderFilter); 0156 return localIds.toSet(); 0157 } else { 0158 //Find all enabled collections 0159 Sink::Query query; 0160 query.setType(mCollectionType); 0161 query.filter("enabled", {true}); 0162 return resolveQuery(query).toSet(); 0163 } 0164 }(); 0165 if (collectionsToSync.isEmpty()) { 0166 SinkTraceCtx(mLogCtx) << "No collections to sync:" << query; 0167 return KAsync::null(); 0168 } 0169 SinkTraceCtx(mLogCtx) << "Synchronizing collections: " << collectionsToSync; 0170 0171 return runJob<KDAV2::DavCollection::List>(new KDAV2::DavCollectionsFetchJob{ serverUrl }, 0172 [](KJob *job) { return static_cast<KDAV2::DavCollectionsFetchJob *>(job)->collections(); }) 0173 .serialEach([=](const KDAV2::DavCollection &collection) { 0174 const auto collectionRid = resourceID(collection); 0175 const auto localId = syncStore().resolveRemoteId(mCollectionType, collectionRid); 0176 //Filter list of folders to sync 0177 if (!collectionsToSync.contains(localId)) { 0178 return KAsync::null(); 0179 } 0180 return synchronizeCollection(collection.url(), collectionRid, localId, collection.CTag().toLatin1()) 0181 .then([=] (const KAsync::Error &error) { 0182 if (error) { 0183 SinkWarningCtx(mLogCtx) << "Failed to synchronized folder" << error; 0184 } 0185 //Ignore synchronization errors for individual collections, the next one might work. 0186 return KAsync::null(); 0187 }); 0188 }); 0189 } else { 0190 SinkWarning() << "Unknown query type" << query; 0191 return KAsync::null<void>(); 0192 } 0193 0194 }); 0195 0196 } 0197 0198 KAsync::Job<void> WebDavSynchronizer::synchronizeCollection(const KDAV2::DavUrl &collectionUrl, const QByteArray &collectionRid, const QByteArray &collectionLocalId, const QByteArray &ctag) 0199 { 0200 auto progress = QSharedPointer<int>::create(0); 0201 auto total = QSharedPointer<int>::create(0); 0202 if (ctag == syncStore().readValue(collectionRid + "_ctag")) { 0203 SinkTraceCtx(mLogCtx) << "Collection unchanged:" << collectionRid; 0204 return KAsync::null<void>(); 0205 } 0206 SinkLogCtx(mLogCtx) << "Syncing collection:" << collectionRid << ctag << collectionUrl; 0207 0208 auto itemsResourceIDs = QSharedPointer<QSet<QByteArray>>::create(); 0209 0210 auto listJob = new KDAV2::DavItemsListJob(collectionUrl); 0211 if (mCollectionType == "calendar") { 0212 listJob->setContentMimeTypes({{"VEVENT"}, {"VTODO"}}); 0213 } 0214 return runJob<KDAV2::DavItem::List>(listJob, 0215 [](KJob *job) { return static_cast<KDAV2::DavItemsListJob *>(job)->items(); }) 0216 .then([=](const KDAV2::DavItem::List &items) { 0217 SinkLogCtx(mLogCtx) << "Found" << items.size() << "items on the server"; 0218 QStringList itemsToFetch; 0219 for (const auto &item : items) { 0220 const auto itemRid = resourceID(item); 0221 itemsResourceIDs->insert(itemRid); 0222 if (item.etag().toLatin1() == syncStore().readValue(collectionRid, itemRid + "_etag")) { 0223 SinkTraceCtx(mLogCtx) << "Item unchanged:" << itemRid; 0224 continue; 0225 } 0226 itemsToFetch << item.url().url().toDisplayString(); 0227 } 0228 if (itemsToFetch.isEmpty()) { 0229 return KAsync::null(); 0230 } 0231 *total += itemsToFetch.size(); 0232 return runJob<KDAV2::DavItem::List>(new KDAV2::DavItemsFetchJob(collectionUrl, itemsToFetch), 0233 [](KJob *job) { return static_cast<KDAV2::DavItemsFetchJob *>(job)->items(); }) 0234 .then([=] (const KDAV2::DavItem::List &items) { 0235 for (const auto &item : items) { 0236 updateLocalItem(item, collectionLocalId); 0237 syncStore().writeValue(collectionRid, resourceID(item) + "_etag", item.etag().toLatin1()); 0238 } 0239 0240 }); 0241 }) 0242 .then([=] { 0243 // Update the local CTag to be able to tell if the collection is unchanged 0244 syncStore().writeValue(collectionRid + "_ctag", ctag); 0245 0246 for (const auto &entityType : mEntityTypes) { 0247 int count = scanForRemovals(entityType, 0248 [&](const std::function<void(const QByteArray &)> &callback) { 0249 //FIXME: The collection type just happens to have the same name as the parent collection property 0250 const auto collectionProperty = mCollectionType; 0251 store().indexLookup(entityType, collectionProperty, collectionLocalId, callback); 0252 }, 0253 [&itemsResourceIDs](const QByteArray &remoteId) { 0254 return itemsResourceIDs->contains(remoteId); 0255 }); 0256 SinkLogCtx(mLogCtx) << "Removed " << count << " items"; 0257 } 0258 }); 0259 } 0260 0261 KAsync::Job<KDAV2::DavUrl> WebDavSynchronizer::discoverServer() 0262 { 0263 if (mCachedServer.url().isValid()) { 0264 return KAsync::value(mCachedServer); 0265 } 0266 if (!mServer.isValid()) { 0267 return KAsync::error<KDAV2::DavUrl>(Sink::ApplicationDomain::ConfigurationError, "Invalid server url: " + mServer.toString()); 0268 } 0269 0270 if (secret().isEmpty()) { 0271 return KAsync::error<KDAV2::DavUrl>(Sink::ApplicationDomain::ConfigurationError, "No secret"); 0272 } 0273 0274 auto result = mServer; 0275 result.setUserName(mUsername); 0276 result.setPassword(secret()); 0277 const KDAV2::DavUrl serverUrl{result, mProtocol}; 0278 0279 return runJob<KDAV2::DavUrl>(new KDAV2::DavDiscoveryJob(serverUrl, mCollectionType == "addressbook" ? "carddav" : "caldav"), [=] (KJob *job) { 0280 auto url = serverUrl; 0281 url.setUrl(static_cast<KDAV2::DavDiscoveryJob*>(job)->url()); 0282 mCachedServer = url; 0283 return url; 0284 }); 0285 } 0286 0287 KAsync::Job<QPair<QUrl, QStringList>> WebDavSynchronizer::discoverHome(const KDAV2::DavUrl &serverUrl) 0288 { 0289 return runJob<QPair<QUrl, QStringList>>(new KDAV2::DavPrincipalHomeSetsFetchJob(serverUrl), [=] (KJob *job) { 0290 return qMakePair(static_cast<KDAV2::DavPrincipalHomeSetsFetchJob*>(job)->url(), static_cast<KDAV2::DavPrincipalHomeSetsFetchJob*>(job)->homeSets()); 0291 }); 0292 } 0293 0294 KAsync::Job<QByteArray> WebDavSynchronizer::createItem(const QByteArray &vcard, const QByteArray &contentType, const QByteArray &rid, const QByteArray &collectionRid) 0295 { 0296 return discoverServer() 0297 .then([=] (const KDAV2::DavUrl &serverUrl) { 0298 KDAV2::DavItem remoteItem; 0299 remoteItem.setData(vcard); 0300 remoteItem.setContentType(contentType); 0301 remoteItem.setUrl(urlOf(serverUrl, collectionRid, rid)); 0302 SinkLogCtx(mLogCtx) << "Creating:" << "Rid: " << rid << "Content-Type: " << contentType << "Url: " << remoteItem.url().url() << "Content:\n" << vcard; 0303 0304 return runJob<KDAV2::DavItem>(new KDAV2::DavItemCreateJob(remoteItem), [](KJob *job) { return static_cast<KDAV2::DavItemCreateJob*>(job)->item(); }) 0305 .then([=] (const KDAV2::DavItem &remoteItem) { 0306 syncStore().writeValue(collectionRid, resourceID(remoteItem) + "_etag", remoteItem.etag().toLatin1()); 0307 return resourceID(remoteItem); 0308 }); 0309 }); 0310 0311 } 0312 0313 0314 KAsync::Job<QByteArray> WebDavSynchronizer::moveItem(const QByteArray &vcard, const QByteArray &contentType, const QByteArray &rid, const QByteArray &collectionRid, const QByteArray &oldRemoteId) 0315 { 0316 SinkLogCtx(mLogCtx) << "Moving:" << oldRemoteId; 0317 return createItem(vcard, contentType, rid, collectionRid) 0318 .then([=] (const QByteArray &remoteId) { 0319 return removeItem(oldRemoteId) 0320 .then([=] { 0321 return remoteId; 0322 }); 0323 }); 0324 } 0325 0326 KAsync::Job<QByteArray> WebDavSynchronizer::modifyItem(const QByteArray &oldRemoteId, const QByteArray &vcard, const QByteArray &contentType, const QByteArray &collectionRid) 0327 { 0328 return discoverServer() 0329 .then([=] (const KDAV2::DavUrl &serverUrl) { 0330 KDAV2::DavItem remoteItem; 0331 remoteItem.setData(vcard); 0332 remoteItem.setContentType(contentType); 0333 remoteItem.setUrl(urlOf(serverUrl, oldRemoteId)); 0334 remoteItem.setEtag(syncStore().readValue(collectionRid, oldRemoteId + "_etag")); 0335 SinkLogCtx(mLogCtx) << "Modifying:" << "Content-Type: " << contentType << "Url: " << remoteItem.url().url() << "Etag: " << remoteItem.etag() << "Content:\n" << vcard; 0336 0337 return runJob<KDAV2::DavItem>(new KDAV2::DavItemModifyJob(remoteItem), [](KJob *job) { return static_cast<KDAV2::DavItemModifyJob*>(job)->item(); }) 0338 .then([=] (const KAsync::Error &error, const KDAV2::DavItem &fetchedItem) { 0339 if (error) { 0340 if (error.errorCode != Sink::ApplicationDomain::SynchronizationConflictError) { 0341 SinkWarningCtx(mLogCtx) << "Modification failed, but not a conflict."; 0342 return KAsync::error<QByteArray>(error); 0343 } 0344 SinkLogCtx(mLogCtx) << "Fetching server version to resolve conflict during modification"; 0345 return runJob<KDAV2::DavItem>(new KDAV2::DavItemFetchJob(remoteItem), [](KJob *job) { return static_cast<KDAV2::DavItemFetchJob*>(job)->item(); }) 0346 .then([=] (const KDAV2::DavItem &item) { 0347 const auto collectionLocalId = syncStore().resolveRemoteId(mCollectionType, collectionRid); 0348 const auto remoteId = resourceID(item); 0349 //Overwrite the local version with the sever version. 0350 updateLocalItem(item, collectionLocalId); 0351 syncStore().writeValue(collectionRid, remoteId + "_etag", item.etag().toLatin1()); 0352 return KAsync::value(remoteId); 0353 }); 0354 } 0355 const auto remoteId = resourceID(fetchedItem); 0356 Q_ASSERT(remoteId == oldRemoteId); 0357 syncStore().writeValue(collectionRid, remoteId + "_etag", fetchedItem.etag().toLatin1()); 0358 return KAsync::value(remoteId); 0359 }); 0360 }); 0361 } 0362 0363 KAsync::Job<QByteArray> WebDavSynchronizer::removeItem(const QByteArray &oldRemoteId) 0364 { 0365 return discoverServer() 0366 .then([=] (const KDAV2::DavUrl &serverUrl) { 0367 SinkLogCtx(mLogCtx) << "Removing:" << oldRemoteId; 0368 // We only need the URL in the DAV item for removal 0369 KDAV2::DavItem remoteItem; 0370 remoteItem.setUrl(urlOf(serverUrl, oldRemoteId)); 0371 return runJob(new KDAV2::DavItemDeleteJob(remoteItem)) 0372 .then([] { 0373 return QByteArray{}; 0374 }); 0375 }); 0376 } 0377 0378 KAsync::Job<QByteArray> WebDavSynchronizer::createCollection(const KDAV2::DavCollection &collection, const KDAV2::Protocol protocol) 0379 { 0380 return discoverServer() 0381 .then([=] (const KDAV2::DavUrl &serverUrl) { 0382 return discoverHome(serverUrl) 0383 .then([=] (const QPair<QUrl, QStringList> &pair) { 0384 const auto home = pair.second.first(); 0385 0386 auto url = serverUrl.url(); 0387 url.setPath(home + collection.displayName()); 0388 0389 auto davUrl = serverUrl; 0390 davUrl.setProtocol(protocol); 0391 davUrl.setUrl(url); 0392 0393 auto col = collection; 0394 col.setUrl(davUrl); 0395 SinkLogCtx(mLogCtx) << "Creating collection"<< col.displayName() << col.url() << col.contentTypes(); 0396 auto job = new KDAV2::DavCollectionCreateJob(col); 0397 return runJob(job) 0398 .then([=] { 0399 SinkLogCtx(mLogCtx) << "Done creating collection"; 0400 return resourceID(job->collection()); 0401 }); 0402 }); 0403 }); 0404 } 0405 0406 KAsync::Job<QByteArray> WebDavSynchronizer::removeCollection(const QByteArray &collectionRid) 0407 { 0408 return discoverServer() 0409 .then([=] (const KDAV2::DavUrl &serverUrl) { 0410 return runJob(new KDAV2::DavCollectionDeleteJob(urlOf(serverUrl, collectionRid))) 0411 .then([this] { 0412 SinkLogCtx(mLogCtx) << "Done removing collection"; 0413 return QByteArray{}; 0414 }); 0415 }); 0416 } 0417 0418 KAsync::Job<QByteArray> WebDavSynchronizer::modifyCollection(const QByteArray &collectionRid, const KDAV2::DavCollection &collection) 0419 { 0420 return discoverServer() 0421 .then([=] (const KDAV2::DavUrl &serverUrl) { 0422 auto job = new KDAV2::DavCollectionModifyJob(urlOf(serverUrl, collectionRid)); 0423 0424 //TODO we should be setting those properties in KDAV2 0425 job->setProperty("calendar-color", collection.color().name(), QStringLiteral("http://apple.com/ns/ical/")); 0426 job->setProperty("displayname", collection.displayName(), QStringLiteral("DAV:")); 0427 0428 return runJob(job) 0429 .then([=] { 0430 SinkLogCtx(mLogCtx) << "Done modifying collection"; 0431 return collectionRid; 0432 }); 0433 }); 0434 } 0435 0436 QByteArray WebDavSynchronizer::resourceID(const KDAV2::DavCollection &collection) 0437 { 0438 return collection.url().url().path().toUtf8(); 0439 } 0440 0441 QByteArray WebDavSynchronizer::resourceID(const KDAV2::DavItem &item) 0442 { 0443 return item.url().url().path().toUtf8(); 0444 } 0445 0446 KDAV2::DavUrl WebDavSynchronizer::urlOf(const KDAV2::DavUrl &serverUrl, const QByteArray &remoteId) 0447 { 0448 auto davurl = serverUrl; 0449 auto url = davurl.url(); 0450 url.setPath(remoteId); 0451 davurl.setUrl(url); 0452 return davurl; 0453 } 0454 0455 KDAV2::DavUrl WebDavSynchronizer::urlOf(const KDAV2::DavUrl &serverUrl, const QByteArray &collectionRemoteId, const QString &itemPath) 0456 { 0457 return urlOf(serverUrl, collectionRemoteId + itemPath.toUtf8()); 0458 }