kimap/src/fetchjob.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2009 Kevin Ottens <ervin@kde.org>
0003
0004     SPDX-License-Identifier: LGPL-2.0-or-later
0005 */
0006
0007 #include "fetchjob.h"
0008
0009 #include "kimap_debug.h"
0010 #include <KLocalizedString>
0011 #include <QTimer>
0012
0013 #include "job_p.h"
0014 #include "response_p.h"
0015 #include "session_p.h"
0016
0017 namespace KIMAP
0018 {
0019 class FetchJobPrivate : public JobPrivate
0020 {
0021 public:
0022     FetchJobPrivate(FetchJob *job, Session *session, const QString &name)
0023         : JobPrivate(session, name)
0024         , q(job)
0025     {
0026     }
0027
0028     ~FetchJobPrivate()
0029     {
0030     }
0031
0032     void parseBodyStructure(const QByteArray &structure, int &pos, KMime::Content *content);
0033     void parsePart(const QByteArray &structure, int &pos, KMime::Content *content);
0034     QByteArray parseString(const QByteArray &structure, int &pos);
0035     QByteArray parseSentence(const QByteArray &structure, int &pos);
0036     void skipLeadingSpaces(const QByteArray &structure, int &pos);
0037
0038     void emitPendings()
0039     {
0040         if (pendingMsgs.isEmpty()) {
0041             return;
0042         }
0043
0044         Q_EMIT q->messagesAvailable(pendingMsgs);
0045
0046         if (!pendingParts.isEmpty()) {
0047             Q_EMIT q->partsReceived(selectedMailBox, pendingUids, pendingParts);
0048             Q_EMIT q->partsReceived(selectedMailBox, pendingUids, pendingAttributes, pendingParts);
0049         }
0050         if (!pendingSizes.isEmpty() || !pendingFlags.isEmpty() || !pendingMessages.isEmpty()) {
0051             Q_EMIT q->headersReceived(selectedMailBox, pendingUids, pendingSizes, pendingFlags, pendingMessages);
0052             Q_EMIT q->headersReceived(selectedMailBox, pendingUids, pendingSizes, pendingAttributes, pendingFlags, pendingMessages);
0053         }
0054         if (!pendingMessages.isEmpty()) {
0055             Q_EMIT q->messagesReceived(selectedMailBox, pendingUids, pendingMessages);
0056             Q_EMIT q->messagesReceived(selectedMailBox, pendingUids, pendingAttributes, pendingMessages);
0057         }
0058
0059         pendingUids.clear();
0060         pendingMessages.clear();
0061         pendingParts.clear();
0062         pendingSizes.clear();
0063         pendingFlags.clear();
0064         pendingAttributes.clear();
0065         pendingMsgs.clear();
0066     }
0067
0068     FetchJob *const q;
0069
0070     ImapSet set;
0071     bool uidBased = false;
0072     FetchJob::FetchScope scope;
0073     QString selectedMailBox;
0074     bool gmailEnabled = false;
0075
0076     QTimer emitPendingsTimer;
0077     QMap<qint64, MessagePtr> pendingMessages;
0078     QMap<qint64, MessageParts> pendingParts;
0079     QMap<qint64, MessageFlags> pendingFlags;
0080     QMap<qint64, MessageAttribute> pendingAttributes;
0081     QMap<qint64, qint64> pendingSizes;
0082     QMap<qint64, qint64> pendingUids;
0083     QMap<qint64, Message> pendingMsgs;
0084 };
0085 }
0086
0087 using namespace KIMAP;
0088
0089 FetchJob::FetchScope::FetchScope()
0090     : mode(FetchScope::Content)
0091     , changedSince(0)
0092     , qresync(false)
0093 {
0094 }
0095
0096 FetchJob::FetchJob(Session *session)
0097     : Job(*new FetchJobPrivate(this, session, i18n("Fetch")))
0098 {
0099     Q_D(FetchJob);
0100     connect(&d->emitPendingsTimer, &QTimer::timeout, this, [d]() {
0101         d->emitPendings();
0102     });
0103 }
0104
0105 void FetchJob::setSequenceSet(const ImapSet &set)
0106 {
0107     Q_D(FetchJob);
0108     Q_ASSERT(!set.isEmpty());
0109     d->set = set;
0110 }
0111
0112 ImapSet FetchJob::sequenceSet() const
0113 {
0114     Q_D(const FetchJob);
0115     return d->set;
0116 }
0117
0118 void FetchJob::setUidBased(bool uidBased)
0119 {
0120     Q_D(FetchJob);
0121     d->uidBased = uidBased;
0122 }
0123
0124 bool FetchJob::isUidBased() const
0125 {
0126     Q_D(const FetchJob);
0127     return d->uidBased;
0128 }
0129
0130 void FetchJob::setScope(const FetchScope &scope)
0131 {
0132     Q_D(FetchJob);
0133     d->scope = scope;
0134 }
0135
0136 FetchJob::FetchScope FetchJob::scope() const
0137 {
0138     Q_D(const FetchJob);
0139     return d->scope;
0140 }
0141
0142 bool FetchJob::setGmailExtensionsEnabled() const
0143 {
0144     Q_D(const FetchJob);
0145     return d->gmailEnabled;
0146 }
0147
0148 void FetchJob::setGmailExtensionsEnabled(bool enabled)
0149 {
0150     Q_D(FetchJob);
0151     d->gmailEnabled = enabled;
0152 }
0153
0154 QString FetchJob::mailBox() const
0155 {
0156     Q_D(const FetchJob);
0157     return d->selectedMailBox;
0158 }
0159
0160 void FetchJob::doStart()
0161 {
0162     Q_D(FetchJob);
0163
0164     d->set.optimize();
0165     QByteArray parameters = d->set.toImapSequenceSet() + ' ';
0166     Q_ASSERT(!parameters.trimmed().isEmpty());
0167
0168     switch (d->scope.mode) {
0169     case FetchScope::Headers:
0170         if (d->scope.parts.isEmpty()) {
0171             parameters += "(RFC822.SIZE INTERNALDATE BODY.PEEK[HEADER.FIELDS (TO FROM MESSAGE-ID REFERENCES IN-REPLY-TO SUBJECT DATE)] FLAGS UID";
0172         } else {
0173             parameters += '(';
0174             for (const QByteArray &part : std::as_const(d->scope.parts)) {
0175                 parameters += "BODY.PEEK[" + part + ".MIME] ";
0176             }
0177             parameters += "UID";
0178         }
0179         break;
0180     case FetchScope::Flags:
0181         parameters += "(FLAGS UID";
0182         break;
0183     case FetchScope::Structure:
0184         parameters += "(BODYSTRUCTURE UID";
0185         break;
0186     case FetchScope::Content:
0187         if (d->scope.parts.isEmpty()) {
0188             parameters += "(BODY.PEEK[] UID";
0189         } else {
0190             parameters += '(';
0191             for (const QByteArray &part : std::as_const(d->scope.parts)) {
0192                 parameters += "BODY.PEEK[" + part + "] ";
0193             }
0194             parameters += "UID";
0195         }
0196         break;
0197     case FetchScope::Full:
0198         parameters += "(RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID";
0199         break;
0200     case FetchScope::HeaderAndContent:
0201         if (d->scope.parts.isEmpty()) {
0202             parameters += "(BODY.PEEK[] FLAGS UID";
0203         } else {
0204             parameters += "(BODY.PEEK[HEADER.FIELDS (TO FROM MESSAGE-ID REFERENCES IN-REPLY-TO SUBJECT DATE)]";
0205             for (const QByteArray &part : std::as_const(d->scope.parts)) {
0206                 parameters += " BODY.PEEK[" + part + ".MIME] BODY.PEEK[" + part + "]"; // krazy:exclude=doublequote_chars
0207             }
0208             parameters += " FLAGS UID";
0209         }
0210         break;
0211     case FetchScope::FullHeaders:
0212         parameters += "(RFC822.SIZE INTERNALDATE BODY.PEEK[HEADER] FLAGS UID";
0213         break;
0214     }
0215
0216     if (d->gmailEnabled) {
0217         parameters += " X-GM-LABELS X-GM-MSGID X-GM-THRID";
0218     }
0219     parameters += ")";
0220
0221     if (d->scope.changedSince > 0) {
0222         parameters += " (CHANGEDSINCE " + QByteArray::number(d->scope.changedSince);
0223         if (d->scope.qresync) {
0224             parameters += " VANISHED";
0225         }
0226         parameters += ")";
0227     }
0228
0229     QByteArray command = "FETCH";
0230     if (d->uidBased) {
0231         command = "UID " + command;
0232     }
0233
0234     d->emitPendingsTimer.start(100);
0235     d->selectedMailBox = d->m_session->selectedMailBox();
0236     d->tags << d->sessionInternal()->sendCommand(command, parameters);
0237 }
0238
0239 void FetchJob::handleResponse(const Response &response)
0240 {
0241     Q_D(FetchJob);
0242
0243     // We can predict it'll be handled by handleErrorReplies() so stop
0244     // the timer now so that result() will really be the last emitted signal.
0245     if (!response.content.isEmpty() && d->tags.size() == 1 && d->tags.contains(response.content.first().toString())) {
0246         d->emitPendingsTimer.stop();
0247         d->emitPendings();
0248     }
0249
0250     if (handleErrorReplies(response) == NotHandled) {
0251         if (response.content.size() == 4 && response.content[1].toString() == "VANISHED") {
0252             const auto vanishedSet = ImapSet::fromImapSequenceSet(response.content[3].toString());
0253             Q_EMIT messagesVanished(vanishedSet);
0254         } else if (response.content.size() == 4 && response.content[2].toString() == "FETCH" && response.content[3].type() == Response::Part::List) {
0255             const qint64 id = response.content[1].toString().toLongLong();
0256             const QList<QByteArray> content = response.content[3].toList();
0257
0258             Message msg;
0259             MessagePtr message(new KMime::Message);
0260             bool shouldParseMessage = false;
0261             MessageParts parts;
0262
0263             for (QList<QByteArray>::ConstIterator it = content.constBegin(); it != content.constEnd(); ++it) {
0264                 QByteArray str = *it;
0265                 ++it;
0266
0267                 if (it == content.constEnd()) { // Uh oh, message was truncated?
0268                     qCWarning(KIMAP_LOG) << "FETCH reply got truncated, skipping.";
0269                     break;
0270                 }
0271
0272                 if (str == "UID") {
0273                     d->pendingUids[id] = msg.uid = it->toLongLong();
0274                 } else if (str == "RFC822.SIZE") {
0275                     d->pendingSizes[id] = msg.size = it->toLongLong();
0276                 } else if (str == "INTERNALDATE") {
0277                     message->date()->setDateTime(QDateTime::fromString(QLatin1StringView(*it), Qt::RFC2822Date));
0278                 } else if (str == "FLAGS") {
0279                     if ((*it).startsWith('(') && (*it).endsWith(')')) {
0280                         QByteArray str = *it;
0281                         str.chop(1);
0282                         str.remove(0, 1);
0283                         const auto flags = str.split(' ');
0284                         d->pendingFlags[id] = flags;
0285                         msg.flags = flags;
0286                     } else {
0287                         d->pendingFlags[id] << *it;
0288                         msg.flags << *it;
0289                     }
0290                 } else if (str == "X-GM-LABELS") {
0291                     d->pendingAttributes.insert(id, {"X-GM-LABELS", *it});
0292                     msg.attributes.insert("X-GM-LABELS", *it);
0293                 } else if (str == "X-GM-THRID") {
0294                     d->pendingAttributes.insert(id, {"X-GM-THRID", *it});
0295                     msg.attributes.insert("X-GM-THRID", *it);
0296                 } else if (str == "X-GM-MSGID") {
0297                     d->pendingAttributes.insert(id, {"X-GM-MSGID", *it});
0298                     msg.attributes.insert("X-GM-MSGID", *it);
0299                 } else if (str == "BODYSTRUCTURE") {
0300                     int pos = 0;
0301                     d->parseBodyStructure(*it, pos, message.data());
0302                     message->assemble();
0303                     d->pendingMessages[id] = message;
0304                     msg.message = message;
0305                 } else if (str.startsWith("BODY[")) { // krazy:exclude=strings
0306                     if (!str.endsWith(']')) { // BODY[ ... ] might have been split, skip until we find the ]
0307                         while (!(*it).endsWith(']')) {
0308                             ++it;
0309                         }
0310                         ++it;
0311                     }
0312
0313                     int index;
0314                     if ((index = str.indexOf("HEADER")) > 0 || (index = str.indexOf("MIME")) > 0) { // headers
0315                         if (str[index - 1] == '.') {
0316                             QByteArray partId = str.mid(5, index - 6);
0317                             if (!parts.contains(partId)) {
0318                                 parts[partId] = ContentPtr(new KMime::Content);
0319                             }
0320                             parts[partId]->setHead(*it);
0321                             parts[partId]->parse();
0322                             d->pendingParts[id] = parts;
0323                             msg.parts = parts;
0324                         } else {
0325                             message->setHead(*it);
0326                             shouldParseMessage = true;
0327                         }
0328                     } else { // full payload
0329                         if (str == "BODY[]") {
0330                             message->setContent(KMime::CRLFtoLF(*it));
0331                             shouldParseMessage = true;
0332
0333                             d->pendingMessages[id] = message;
0334                             msg.message = message;
0335                         } else {
0336                             QByteArray partId = str.mid(5, str.size() - 6);
0337                             if (!parts.contains(partId)) {
0338                                 parts[partId] = ContentPtr(new KMime::Content);
0339                             }
0340                             parts[partId]->setBody(*it);
0341                             parts[partId]->parse();
0342
0343                             d->pendingParts[id] = parts;
0344                             msg.parts = parts;
0345                         }
0346                     }
0347                 }
0348             }
0349
0350             if (shouldParseMessage) {
0351                 message->parse();
0352             }
0353
0354             // For the headers mode the message is built in several
0355             // steps, hence why we wait it to be done until putting it
0356             // in the pending queue.
0357             if (d->scope.mode == FetchScope::Headers || d->scope.mode == FetchScope::HeaderAndContent || d->scope.mode == FetchScope::FullHeaders) {
0358                 d->pendingMessages[id] = message;
0359                 msg.message = message;
0360             }
0361
0362             d->pendingMsgs[id] = msg;
0363         }
0364     }
0365 }
0366
0367 void FetchJobPrivate::parseBodyStructure(const QByteArray &structure, int &pos, KMime::Content *content)
0368 {
0369     skipLeadingSpaces(structure, pos);
0370
0371     if (structure[pos] != '(') {
0372         return;
0373     }
0374
0375     pos++;
0376
0377     if (structure[pos] != '(') { // simple part
0378         pos--;
0379         parsePart(structure, pos, content);
0380     } else { // multi part
0381         content->contentType()->setMimeType("MULTIPART/MIXED");
0382         while (pos < structure.size() && structure[pos] == '(') {
0383             auto child = new KMime::Content;
0384             content->appendContent(child);
0385             parseBodyStructure(structure, pos, child);
0386             child->assemble();
0387         }
0388
0389         QByteArray subType = parseString(structure, pos);
0390         content->contentType()->setMimeType("MULTIPART/" + subType);
0391
0392         QByteArray parameters = parseSentence(structure, pos); // FIXME: Read the charset
0393         if (parameters.contains("BOUNDARY")) {
0394             content->contentType()->setBoundary(parameters.remove(0, parameters.indexOf("BOUNDARY") + 11).split('\"')[0]);
0395         }
0396
0397         QByteArray disposition = parseSentence(structure, pos);
0398         if (disposition.contains("INLINE")) {
0399             content->contentDisposition()->setDisposition(KMime::Headers::CDinline);
0400         } else if (disposition.contains("ATTACHMENT")) {
0401             content->contentDisposition()->setDisposition(KMime::Headers::CDattachment);
0402         }
0403
0404         parseSentence(structure, pos); // Ditch the body language
0405     }
0406
0407     // Consume what's left
0408     while (pos < structure.size() && structure[pos] != ')') {
0409         skipLeadingSpaces(structure, pos);
0410         parseSentence(structure, pos);
0411         skipLeadingSpaces(structure, pos);
0412     }
0413
0414     pos++;
0415 }
0416
0417 void FetchJobPrivate::parsePart(const QByteArray &structure, int &pos, KMime::Content *content)
0418 {
0419     if (structure[pos] != '(') {
0420         return;
0421     }
0422
0423     pos++;
0424
0425     QByteArray mainType = parseString(structure, pos);
0426     QByteArray subType = parseString(structure, pos);
0427
0428     content->contentType()->setMimeType(mainType + '/' + subType);
0429
0430     parseSentence(structure, pos); // Ditch the parameters... FIXME: Read it to get charset and name
0431     parseString(structure, pos); // ... and the id
0432
0433     content->contentDescription()->from7BitString(parseString(structure, pos));
0434
0435     parseString(structure, pos); // Ditch the encoding too
0436     parseString(structure, pos); // ... and the size
0437     parseString(structure, pos); // ... and the line count
0438
0439     QByteArray disposition = parseSentence(structure, pos);
0440     if (disposition.contains("INLINE")) {
0441         content->contentDisposition()->setDisposition(KMime::Headers::CDinline);
0442     } else if (disposition.contains("ATTACHMENT")) {
0443         content->contentDisposition()->setDisposition(KMime::Headers::CDattachment);
0444     }
0445     if ((content->contentDisposition()->disposition() == KMime::Headers::CDattachment
0446          || content->contentDisposition()->disposition() == KMime::Headers::CDinline)
0447         && disposition.contains("FILENAME")) {
0448         QByteArray filename = disposition.remove(0, disposition.indexOf("FILENAME") + 11).split('\"')[0];
0449         content->contentDisposition()->setFilename(QLatin1StringView(filename));
0450     }
0451
0452     // Consume what's left
0453     while (pos < structure.size() && structure[pos] != ')') {
0454         skipLeadingSpaces(structure, pos);
0455         parseSentence(structure, pos);
0456         skipLeadingSpaces(structure, pos);
0457     }
0458 }
0459
0460 QByteArray FetchJobPrivate::parseSentence(const QByteArray &structure, int &pos)
0461 {
0462     QByteArray result;
0463     int stack = 0;
0464
0465     skipLeadingSpaces(structure, pos);
0466
0467     if (structure[pos] != '(') {
0468         return parseString(structure, pos);
0469     }
0470
0471     int start = pos;
0472
0473     do {
0474         switch (structure[pos]) {
0475         case '(':
0476             pos++;
0477             stack++;
0478             break;
0479         case ')':
0480             pos++;
0481             stack--;
0482             break;
0483         case '[':
0484             pos++;
0485             stack++;
0486             break;
0487         case ']':
0488             pos++;
0489             stack--;
0490             break;
0491         default:
0492             skipLeadingSpaces(structure, pos);
0493             parseString(structure, pos);
0494             skipLeadingSpaces(structure, pos);
0495             break;
0496         }
0497     } while (pos < structure.size() && stack != 0);
0498
0499     result = structure.mid(start, pos - start);
0500
0501     return result;
0502 }
0503
0504 QByteArray FetchJobPrivate::parseString(const QByteArray &structure, int &pos)
0505 {
0506     QByteArray result;
0507
0508     skipLeadingSpaces(structure, pos);
0509
0510     int start = pos;
0511     bool foundSlash = false;
0512
0513     // quoted string
0514     if (structure[pos] == '"') {
0515         pos++;
0516         for (;;) {
0517             if (structure[pos] == '\\') {
0518                 pos += 2;
0519                 foundSlash = true;
0520                 continue;
0521             }
0522             if (structure[pos] == '"') {
0523                 result = structure.mid(start + 1, pos - start - 1);
0524                 pos++;
0525                 break;
0526             }
0527             pos++;
0528         }
0529     } else { // unquoted string
0530         for (;;) {
0531             if (structure[pos] == ' ' || structure[pos] == '(' || structure[pos] == ')' || structure[pos] == '[' || structure[pos] == ']'
0532                 || structure[pos] == '\n' || structure[pos] == '\r' || structure[pos] == '"') {
0533                 break;
0534             }
0535             if (structure[pos] == '\\') {
0536                 foundSlash = true;
0537             }
0538             pos++;
0539         }
0540
0541         result = structure.mid(start, pos - start);
0542
0543         // transform unquoted NIL
0544         if (result == "NIL") {
0545             result.clear();
0546         }
0547     }
0548
0549     // simplify slashes
0550     if (foundSlash) {
0551         while (result.contains("\\\"")) {
0552             result.replace("\\\"", "\"");
0553         }
0554         while (result.contains("\\\\")) {
0555             result.replace("\\\\", "\\");
0556         }
0557     }
0558
0559     return result;
0560 }
0561
0562 void FetchJobPrivate::skipLeadingSpaces(const QByteArray &structure, int &pos)
0563 {
0564     while (pos < structure.size() && structure[pos] == ' ') {
0565         pos++;
0566     }
0567 }
0568
0569 #include "moc_fetchjob.cpp"