sink/tests/mailthreadtest.cpp

0001 /*
0002  *   Copyright (C) 2016 Christian Mollekopf <chrigi_1@fastmail.fm>
0003  *
0004  *   This program is free software; you can redistribute it and/or modify
0005  *   it under the terms of the GNU General Public License as published by
0006  *   the Free Software Foundation; either version 2 of the License, or
0007  *   (at your option) any later version.
0008  *
0009  *   This program is distributed in the hope that it will be useful,
0010  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
0011  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0012  *   GNU General Public License for more details.
0013  *
0014  *   You should have received a copy of the GNU General Public License
0015  *   along with this program; if not, write to the
0016  *   Free Software Foundation, Inc.,
0017  *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
0018  */
0019 #include "mailthreadtest.h"
0020
0021 #include <QTest>
0022
0023 #include <QString>
0024 #include <QFile>
0025 #include <KMime/Message>
0026
0027 #include "store.h"
0028 #include "resourcecontrol.h"
0029 #include "log.h"
0030 #include "test.h"
0031 #include "standardqueries.h"
0032 #include "index.h"
0033 #include "definitions.h"
0034
0035 using namespace Sink;
0036 using namespace Sink::ApplicationDomain;
0037
0038 //TODO extract resource test
0039 //
0040 void MailThreadTest::initTestCase()
0041 {
0042     Test::initTest();
0043     QVERIFY(isBackendAvailable());
0044     resetTestEnvironment();
0045     auto resource = createResource();
0046     QVERIFY(!resource.identifier().isEmpty());
0047
0048     VERIFYEXEC(Store::create(resource));
0049
0050     mResourceInstanceIdentifier = resource.identifier();
0051     mCapabilities = resource.getProperty("capabilities").value<QByteArrayList>();
0052 }
0053
0054 void MailThreadTest::cleanup()
0055 {
0056     VERIFYEXEC(ResourceControl::shutdown(mResourceInstanceIdentifier));
0057     removeResourceFromDisk(mResourceInstanceIdentifier);
0058 }
0059
0060 void MailThreadTest::init()
0061 {
0062     VERIFYEXEC(ResourceControl::start(mResourceInstanceIdentifier));
0063 }
0064
0065
0066 void MailThreadTest::testListThreadLeader()
0067 {
0068     Sink::Query query;
0069     query.resourceFilter(mResourceInstanceIdentifier);
0070     query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0071     query.sort<Mail::Date>();
0072     query.reduce<Mail::ThreadId>(Query::Reduce::Selector::max<Mail::Date>()).count("count").collect<Mail::Sender>("senders");
0073
0074     // Ensure all local data is processed
0075     VERIFYEXEC(Store::synchronize(query));
0076     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
0077
0078     auto mails = Store::read<Mail>(query);
0079     QCOMPARE(mails.size(), 1);
0080     QVERIFY(mails.first().getSubject().startsWith(QString("ThreadLeader")));
0081     auto threadSize = mails.first().getProperty("count").toInt();
0082     QCOMPARE(threadSize, 2);
0083     QCOMPARE(mails.first().aggregatedIds().size(), 2);
0084 }
0085
0086 /*
0087  * Thread:
0088  * 1.
0089  *  2.
0090  *   3.
0091  *
0092  * 3. first, should result in a new thread.
0093  * 1. second, should be merged by subject
0094  * 2. last, should complete the thread.
0095  */
0096 void MailThreadTest::testIndexInMixedOrder()
0097 {
0098     auto folder = Folder::create(mResourceInstanceIdentifier);
0099     folder.setName("folder");
0100     VERIFYEXEC(Store::create(folder));
0101
0102     auto message1 = KMime::Message::Ptr::create();
0103     message1->subject(true)->fromUnicodeString("1", "utf8");
0104     message1->messageID(true)->generate("foobar.com");
0105     message1->date(true)->setDateTime(QDateTime::currentDateTimeUtc());
0106     message1->assemble();
0107
0108     auto message2 = KMime::Message::Ptr::create();
0109     message2->subject(true)->fromUnicodeString("Re: 1", "utf8");
0110     message2->messageID(true)->generate("foobar.com");
0111     message2->inReplyTo(true)->appendIdentifier(message1->messageID(true)->identifier());
0112     message2->date(true)->setDateTime(QDateTime::currentDateTimeUtc().addSecs(1));
0113     message2->assemble();
0114
0115     auto message3 = KMime::Message::Ptr::create();
0116     message3->subject(true)->fromUnicodeString("Re: Re: 1", "utf8");
0117     message3->messageID(true)->generate("foobar.com");
0118     message3->inReplyTo(true)->appendIdentifier(message2->messageID(true)->identifier());
0119     message3->date(true)->setDateTime(QDateTime::currentDateTimeUtc().addSecs(2));
0120     message3->assemble();
0121
0122     {
0123         auto mail = Mail::create(mResourceInstanceIdentifier);
0124         mail.setMimeMessage(message3->encodedContent(true));
0125         mail.setFolder(folder);
0126         VERIFYEXEC(Store::create(mail));
0127     }
0128     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
0129
0130     auto query = Sink::StandardQueries::threadLeaders(folder);
0131     query.resourceFilter(mResourceInstanceIdentifier);
0132     query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0133
0134     Mail threadLeader;
0135
0136     //Ensure we find the thread leader
0137     {
0138         auto mails = Store::read<Mail>(query);
0139         QCOMPARE(mails.size(), 1);
0140         auto mail = mails.first();
0141         threadLeader = mail;
0142         QCOMPARE(mail.getSubject(), QString::fromLatin1("Re: Re: 1"));
0143     }
0144
0145     {
0146         auto mail = Mail::create(mResourceInstanceIdentifier);
0147         mail.setMimeMessage(message2->encodedContent(true));
0148         mail.setFolder(folder);
0149         VERIFYEXEC(Store::create(mail));
0150     }
0151     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
0152
0153     //Ensure we find the thread leader still
0154     {
0155         auto mails = Store::read<Mail>(query);
0156         QCOMPARE(mails.size(), 1);
0157         auto mail = mails.first();
0158         QCOMPARE(mail.getSubject(), QString::fromLatin1("Re: Re: 1"));
0159     }
0160
0161     {
0162         auto mail = Mail::create(mResourceInstanceIdentifier);
0163         mail.setMimeMessage(message1->encodedContent(true));
0164         mail.setFolder(folder);
0165         VERIFYEXEC(Store::create(mail));
0166     }
0167     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
0168
0169     //Ensure the thread is complete
0170     {
0171         auto query = Sink::StandardQueries::completeThread(threadLeader);
0172         query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0173
0174         auto mails = Store::read<Mail>(query);
0175         QCOMPARE(mails.size(), 3);
0176         auto mail = mails.first();
0177         QCOMPARE(mail.getSubject(), QString::fromLatin1("Re: Re: 1"));
0178     }
0179
0180     /* VERIFYEXEC(Store::remove(mail)); */
0181     /* VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); */
0182     /* { */
0183     /*     auto job = Store::fetchAll<Mail>(Query::RequestedProperties(QByteArrayList() << Mail::Folder::name << Mail::Subject::name)) */
0184     /*         .then([=](const QList<Mail::Ptr> &mails) { */
0185     /*             QCOMPARE(mails.size(), 0); */
0186     /*         }); */
0187     /*     VERIFYEXEC(job); */
0188     /* } */
0189     /* VERIFYEXEC(ResourceControl::flushReplayQueue(QByteArrayList() << mResourceInstanceIdentifier)); */
0190 }
0191
0192 static QByteArray readMailFromFile(const QString &mailFile)
0193 {
0194     QFile file(QLatin1String(THREADTESTDATAPATH) + QLatin1Char('/') + mailFile);
0195     file.open(QIODevice::ReadOnly);
0196     Q_ASSERT(file.isOpen());
0197     return file.readAll();
0198 }
0199
0200 static KMime::Message::Ptr readMail(const QString &mailFile)
0201 {
0202     auto msg = KMime::Message::Ptr::create();
0203     msg->setContent(readMailFromFile(mailFile));
0204     msg->parse();
0205     return msg;
0206 }
0207
0208 void MailThreadTest::testRealWorldThread()
0209 {
0210     auto folder = Folder::create(mResourceInstanceIdentifier);
0211     folder.setName("folder");
0212     VERIFYEXEC(Store::create(folder));
0213
0214     auto createMail = [this, folder] (KMime::Message::Ptr msg) {
0215         auto mail = Mail::create(mResourceInstanceIdentifier);
0216         mail.setMimeMessage(msg->encodedContent(true));
0217         mail.setFolder(folder);
0218         VERIFYEXEC(Store::create(mail));
0219     };
0220
0221     createMail(readMail("thread1_1"));
0222
0223     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
0224
0225     auto query = Sink::StandardQueries::threadLeaders(folder);
0226     query.resourceFilter(mResourceInstanceIdentifier);
0227     query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0228
0229     //Ensure we find the thread leader
0230     Mail threadLeader = [&] {
0231         auto mails = Store::read<Mail>(query);
0232         Q_ASSERT(mails.size() == 1);
0233         return mails.first();
0234     }();
0235
0236     createMail(readMail("thread1_2"));
0237     createMail(readMail("thread1_3"));
0238     createMail(readMail("thread1_4"));
0239     createMail(readMail("thread1_5"));
0240     createMail(readMail("thread1_6"));
0241     createMail(readMail("thread1_7"));
0242     createMail(readMail("thread1_8")); //This mail is breaking the thread
0243     VERIFYEXEC(ResourceControl::flushMessageQueue(mResourceInstanceIdentifier));
0244
0245     //Ensure the thread is complete
0246     {
0247         auto query = Sink::StandardQueries::completeThread(threadLeader);
0248         query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0249
0250         auto mails = Store::read<Mail>(query);
0251         QCOMPARE(mails.size(), 8);
0252     }
0253
0254     {
0255         auto query = Sink::StandardQueries::threadLeaders(folder);
0256         Mail threadLeader2 = [&] {
0257             auto mails = Store::read<Mail>(query);
0258             Q_ASSERT(mails.size() == 1);
0259             return mails.first();
0260         }();
0261
0262         {
0263             auto query = Sink::StandardQueries::completeThread(threadLeader2);
0264             query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>();
0265
0266             auto mails = Store::read<Mail>(query);
0267             QCOMPARE(mails.size(), 8);
0268         }
0269     }
0270 }
0271
0272 //Avoid accidentally merging or changing threads
0273 void MailThreadTest::testNoParentsWithModifications()
0274 {
0275     auto folder = Folder::create(mResourceInstanceIdentifier);
0276     folder.setName("folder2");
0277     VERIFYEXEC(Store::create(folder));
0278
0279     auto createMail = [&] (const QString &subject) {
0280         auto message1 = KMime::Message::Ptr::create();
0281         message1->subject(true)->fromUnicodeString(subject, "utf8");
0282         message1->messageID(true)->fromUnicodeString("<" + subject + "@foobar.com" + ">", "utf8");
0283         message1->date(true)->setDateTime(QDateTime::currentDateTimeUtc());
0284         message1->assemble();
0285
0286         auto mail = Mail::create(mResourceInstanceIdentifier);
0287         mail.setMimeMessage(message1->encodedContent(true));
0288         mail.setFolder(folder);
0289         return mail;
0290     };
0291
0292     auto mail1 = createMail("1");
0293     VERIFYEXEC(Store::create(mail1));
0294     auto mail2 = createMail("2");
0295     VERIFYEXEC(Store::create(mail2));
0296     VERIFYEXEC(ResourceControl::flushMessageQueue(mResourceInstanceIdentifier));
0297
0298     auto query = Sink::StandardQueries::threadLeaders(folder);
0299     query.resourceFilter(mResourceInstanceIdentifier);
0300     query.request<Mail::Subject>().request<Mail::MimeMessage>().request<Mail::Folder>().request<Mail::Date>().request<Mail::ThreadId>();
0301
0302     QSet<QByteArray> threadIds;
0303     {
0304         auto mails = Store::read<Mail>(query);
0305         QCOMPARE(mails.size(), 2);
0306         for (const auto &m : mails) {
0307             threadIds << m.getProperty(Mail::ThreadId::name).toByteArray();
0308         }
0309     }
0310
0311     auto readIndex = [&] (const QString &indexName, const QByteArray &lookupKey) {
0312         Index index(Sink::storageLocation(), mResourceInstanceIdentifier, indexName, Sink::Storage::DataStore::ReadOnly);
0313         QByteArrayList keys;
0314         index.lookup(lookupKey,
0315             [&](const QByteArray &value) { keys << QByteArray{value.constData(), value.size()}; return true; },
0316             [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message; },
0317             false);
0318         return keys;
0319     };
0320     QCOMPARE(readIndex("mail.index.messageIdthreadId", "1@foobar.com").size(), 1);
0321     QCOMPARE(readIndex("mail.index.messageIdthreadId", "2@foobar.com").size(), 1);
0322
0323     //We try to modify both mails on purpose
0324     auto checkMail = [&] (Mail mail1) {
0325         Mail modification = mail1;
0326         modification.setChangedProperties({});
0327         modification.setImportant(true);
0328         VERIFYEXEC(Store::modify(modification));
0329         VERIFYEXEC(ResourceControl::flushMessageQueue(mResourceInstanceIdentifier));
0330
0331         QCOMPARE(readIndex("mail.index.messageIdthreadId", "1@foobar.com").size(), 1);
0332         QCOMPARE(readIndex("mail.index.messageIdthreadId", "2@foobar.com").size(), 1);
0333
0334         {
0335             auto mails = Store::read<Mail>(query);
0336             QCOMPARE(mails.size(), 2);
0337             QSet<QByteArray> newThreadIds;
0338             for (const auto &m : mails) {
0339                 newThreadIds << m.getProperty(Mail::ThreadId::name).toByteArray();
0340             }
0341             QCOMPARE(threadIds, newThreadIds);
0342         }
0343     };
0344     checkMail(mail1);
0345     checkMail(mail2);
0346 }
0347
0348
0349 void MailThreadTest::testRealWorldThread2()
0350 {
0351     auto folder = Folder::create(mResourceInstanceIdentifier);
0352     folder.setName("folder2");
0353     VERIFYEXEC(Store::create(folder));
0354
0355     auto createMail = [this, folder] (KMime::Message::Ptr msg) {
0356         auto mail = Mail::create(mResourceInstanceIdentifier);
0357         mail.setMimeMessage(msg->encodedContent(true));
0358         mail.setFolder(folder);
0359         VERIFYEXEC(Store::create(mail));
0360     };
0361
0362     createMail(readMail(QString("thread2_%1").arg(1))); //30.10.18
0363     createMail(readMail(QString("thread2_%1").arg(2))); //02.11.18
0364     createMail(readMail(QString("thread2_%1").arg(3))); //07.11.18
0365     createMail(readMail(QString("thread2_%1").arg(4))); //09.11.18
0366     createMail(readMail(QString("thread2_%1").arg(14))); //13.11.18
0367     createMail(readMail(QString("thread2_%1").arg(12))); //16.11.18
0368     createMail(readMail(QString("thread2_%1").arg(6))); //16.11.18
0369     createMail(readMail(QString("thread2_%1").arg(9))); //23.11.18
0370     // createMail(readMail(QString("thread2_%1").arg(i))); //Different thread 18.1
0371     createMail(readMail(QString("thread2_%1").arg(7))); //04.12.18
0372     createMail(readMail(QString("thread2_%1").arg(17))); //18.12.18
0373     createMail(readMail(QString("thread2_%1").arg(13))); //22.1
0374     createMail(readMail(QString("thread2_%1").arg(15))); //25.1
0375     createMail(readMail(QString("thread2_%1").arg(11))); //28.1
0376     createMail(readMail(QString("thread2_%1").arg(10))); //29.1
0377     createMail(readMail(QString("thread2_%1").arg(16))); //29.1
0378
0379
0380     VERIFYEXEC(ResourceControl::flushMessageQueue(mResourceInstanceIdentifier));
0381
0382     //Ensure we only got one thread
0383     const auto mails = Store::read<Mail>(Sink::StandardQueries::threadLeaders(folder));
0384     QCOMPARE(mails.size(), 1);
0385
0386     //Ensure the thread is complete
0387     QCOMPARE(Store::read<Mail>(Sink::StandardQueries::completeThread(mails.first())).size(), 15);
0388 }
0389