1 /*
2     SPDX-FileCopyrightText: 2010 Klarälvdalens Datakonsult AB, a KDAB Group company <info@kdab.com>
3     SPDX-FileContributor: Kevin Ottens <kevin@kdab.com>
4     SPDX-FileCopyrightText: 2014 Christian Mollekopf <mollekopf@kolabsys.com>
5 
6     SPDX-License-Identifier: LGPL-2.0-or-later
7 */
8 
9 #include "retrieveitemstask.h"
10 
11 #include "batchfetcher.h"
12 #include "collectionflagsattribute.h"
13 #include "highestmodseqattribute.h"
14 #include "messagehelper.h"
15 #include "noselectattribute.h"
16 #include "uidnextattribute.h"
17 #include "uidvalidityattribute.h"
18 
19 #include <Akonadi/AgentBase>
20 #include <Akonadi/CachePolicy>
21 #include <Akonadi/CollectionStatistics>
22 #include <Akonadi/ItemFetchJob>
23 #include <Akonadi/ItemFetchScope>
24 #include <Akonadi/KMime/MessageParts>
25 #include <Akonadi/Session>
26 
27 #include "imapresource_debug.h"
28 
29 #include <KLocalizedString>
30 
31 #include <KIMAP/ExpungeJob>
32 #include <KIMAP/SearchJob>
33 #include <KIMAP/SelectJob>
34 #include <KIMAP/Session>
35 #include <KIMAP/StatusJob>
36 
RetrieveItemsTask(const ResourceStateInterface::Ptr & resource,QObject * parent)37 RetrieveItemsTask::RetrieveItemsTask(const ResourceStateInterface::Ptr &resource, QObject *parent)
38     : ResourceTask(CancelIfNoSession, resource, parent)
39     , m_fetchedMissingBodies(-1)
40 {
41 }
42 
~RetrieveItemsTask()43 RetrieveItemsTask::~RetrieveItemsTask()
44 {
45 }
46 
setFetchMissingItemBodies(bool enabled)47 void RetrieveItemsTask::setFetchMissingItemBodies(bool enabled)
48 {
49     m_fetchMissingBodies = enabled;
50 }
51 
doStart(KIMAP::Session * session)52 void RetrieveItemsTask::doStart(KIMAP::Session *session)
53 {
54     emitPercent(0);
55     // Prevent fetching items from noselect folders.
56     if (collection().hasAttribute("noselect")) {
57         NoSelectAttribute *noselect = static_cast<NoSelectAttribute *>(collection().attribute("noselect"));
58         if (noselect->noSelect()) {
59             qCDebug(IMAPRESOURCE_LOG) << "No Select folder";
60             itemsRetrievalDone();
61             return;
62         }
63     }
64 
65     m_session = session;
66 
67     const Akonadi::Collection col = collection();
68     // Only with emails we can be sure that RID is persistent and thus we can use
69     // it for merging. For other potential content types (like Kolab events etc.)
70     // use GID instead.
71     QStringList cts = col.contentMimeTypes();
72     cts.removeOne(Akonadi::Collection::mimeType());
73     cts.removeOne(KMime::Message::mimeType());
74     if (!cts.isEmpty()) {
75         setItemMergingMode(Akonadi::ItemSync::GIDMerge);
76     } else {
77         setItemMergingMode(Akonadi::ItemSync::RIDMerge);
78     }
79 
80     if (m_fetchMissingBodies
81         && col.cachePolicy().localParts().contains(QLatin1String(Akonadi::MessagePart::Body))) { // disconnected mode, make sure we really have the body cached
82         auto session = new Akonadi::Session(resourceName().toLatin1() + "_body_checker", this);
83         auto fetchJob = new Akonadi::ItemFetchJob(col, session);
84         fetchJob->fetchScope().setCheckForCachedPayloadPartsOnly();
85         fetchJob->fetchScope().fetchPayloadPart(Akonadi::MessagePart::Body);
86         fetchJob->fetchScope().setFetchModificationTime(false);
87         connect(fetchJob, &Akonadi::ItemFetchJob::result, this, &RetrieveItemsTask::fetchItemsWithoutBodiesDone);
88         connect(fetchJob, &Akonadi::ItemFetchJob::result, session, &Akonadi::Session::deleteLater);
89     } else {
90         startRetrievalTasks();
91     }
92 }
93 
createBatchFetcher(MessageHelper::Ptr messageHelper,const KIMAP::ImapSet & set,const KIMAP::FetchJob::FetchScope & scope,int batchSize,KIMAP::Session * session)94 BatchFetcher *RetrieveItemsTask::createBatchFetcher(MessageHelper::Ptr messageHelper,
95                                                     const KIMAP::ImapSet &set,
96                                                     const KIMAP::FetchJob::FetchScope &scope,
97                                                     int batchSize,
98                                                     KIMAP::Session *session)
99 {
100     return new BatchFetcher(messageHelper, set, scope, batchSize, session);
101 }
102 
fetchItemsWithoutBodiesDone(KJob * job)103 void RetrieveItemsTask::fetchItemsWithoutBodiesDone(KJob *job)
104 {
105     QVector<qint64> uids;
106     if (job->error()) {
107         qCWarning(IMAPRESOURCE_LOG) << job->errorString();
108         cancelTask(job->errorString());
109         return;
110     } else {
111         int i = 0;
112         auto fetch = static_cast<Akonadi::ItemFetchJob *>(job);
113         const Akonadi::Item::List lstItems = fetch->items();
114         for (const Akonadi::Item &item : lstItems) {
115             if (!item.cachedPayloadParts().contains(Akonadi::MessagePart::Body)) {
116                 qCWarning(IMAPRESOURCE_LOG) << "Item " << item.id() << " is missing the payload! Cached payloads: " << item.cachedPayloadParts();
117                 uids.append(item.remoteId().toInt());
118                 i++;
119             }
120         }
121         if (i > 0) {
122             qCWarning(IMAPRESOURCE_LOG) << "Number of items missing the body: " << i;
123         }
124     }
125     onFetchItemsWithoutBodiesDone(uids);
126 }
127 
onFetchItemsWithoutBodiesDone(const QVector<qint64> & items)128 void RetrieveItemsTask::onFetchItemsWithoutBodiesDone(const QVector<qint64> &items)
129 {
130     m_messageUidsMissingBody = items;
131     startRetrievalTasks();
132 }
133 
startRetrievalTasks()134 void RetrieveItemsTask::startRetrievalTasks()
135 {
136     const QString mailBox = mailBoxForCollection(collection());
137     qCDebug(IMAPRESOURCE_LOG) << "Starting retrieval for " << mailBox;
138     m_time.start();
139 
140     // Now is the right time to expunge the messages marked \\Deleted from this mailbox.
141     const bool hasACL = serverCapabilities().contains(QLatin1String("ACL"));
142     const KIMAP::Acl::Rights rights = myRights(collection());
143     if (isAutomaticExpungeEnabled() && (!hasACL || (rights & KIMAP::Acl::Expunge) || (rights & KIMAP::Acl::Delete))) {
144         if (m_session->selectedMailBox() != mailBox) {
145             triggerPreExpungeSelect(mailBox);
146         } else {
147             triggerExpunge(mailBox);
148         }
149     } else {
150         // Always select to get the stats updated
151         triggerFinalSelect(mailBox);
152     }
153 }
154 
triggerPreExpungeSelect(const QString & mailBox)155 void RetrieveItemsTask::triggerPreExpungeSelect(const QString &mailBox)
156 {
157     auto select = new KIMAP::SelectJob(m_session);
158     select->setMailBox(mailBox);
159     select->setCondstoreEnabled(serverSupportsCondstore());
160     connect(select, &KJob::result, this, &RetrieveItemsTask::onPreExpungeSelectDone);
161     select->start();
162 }
163 
onPreExpungeSelectDone(KJob * job)164 void RetrieveItemsTask::onPreExpungeSelectDone(KJob *job)
165 {
166     if (job->error()) {
167         qCWarning(IMAPRESOURCE_LOG) << job->errorString();
168         cancelTask(job->errorString());
169     } else {
170         auto select = static_cast<KIMAP::SelectJob *>(job);
171         if (select->isOpenReadOnly()) {
172             qCDebug(IMAPRESOURCE_LOG) << "Mailbox is opened readonly, not expunging";
173             // Treat this SELECT as if it was triggerFinalSelect()
174             onFinalSelectDone(job);
175         } else {
176             triggerExpunge(select->mailBox());
177         }
178     }
179 }
180 
triggerExpunge(const QString & mailBox)181 void RetrieveItemsTask::triggerExpunge(const QString &mailBox)
182 {
183     Q_UNUSED(mailBox)
184     auto expunge = new KIMAP::ExpungeJob(m_session);
185     connect(expunge, &KJob::result, this, &RetrieveItemsTask::onExpungeDone);
186     expunge->start();
187 }
188 
onExpungeDone(KJob * job)189 void RetrieveItemsTask::onExpungeDone(KJob *job)
190 {
191     // We can ignore the error, we just had a wrong expunge so some old messages will just reappear.
192     // TODO we should probably hide messages that are marked as deleted (skipping will not work because we rely on the message count)
193     if (job->error()) {
194         qCWarning(IMAPRESOURCE_LOG) << "Expunge failed: " << job->errorString();
195     }
196     // Except for network errors.
197     if (job->error() && m_session->state() == KIMAP::Session::Disconnected) {
198         cancelTask(job->errorString());
199         return;
200     }
201 
202     // We have to re-select the mailbox to update all the stats after the expunge
203     // (the EXPUNGE command doesn't return enough for our needs)
204     triggerFinalSelect(m_session->selectedMailBox());
205 }
206 
triggerFinalSelect(const QString & mailBox)207 void RetrieveItemsTask::triggerFinalSelect(const QString &mailBox)
208 {
209     auto select = new KIMAP::SelectJob(m_session);
210     select->setMailBox(mailBox);
211     select->setCondstoreEnabled(serverSupportsCondstore());
212     connect(select, &KJob::result, this, &RetrieveItemsTask::onFinalSelectDone);
213     select->start();
214 }
215 
onFinalSelectDone(KJob * job)216 void RetrieveItemsTask::onFinalSelectDone(KJob *job)
217 {
218     auto select = qobject_cast<KIMAP::SelectJob *>(job);
219 
220     if (job->error()) {
221         qCWarning(IMAPRESOURCE_LOG) << select->mailBox() << ":" << job->errorString();
222         cancelTask(select->mailBox() + QStringLiteral(" : ") + job->errorString());
223         return;
224     }
225 
226     m_mailBox = select->mailBox();
227     m_messageCount = select->messageCount();
228     m_uidValidity = select->uidValidity();
229     m_nextUid = select->nextUid();
230     m_highestModSeq = select->highestModSequence();
231     m_flags = select->permanentFlags();
232 
233     // This is known to happen with Courier IMAP.
234     if (m_nextUid < 0) {
235         auto status = new KIMAP::StatusJob(m_session);
236         status->setMailBox(m_mailBox);
237         status->setDataItems({"UIDNEXT"});
238         connect(status, &KJob::result, this, &RetrieveItemsTask::onStatusDone);
239         status->start();
240     } else {
241         prepareRetrieval();
242     }
243 }
244 
onStatusDone(KJob * job)245 void RetrieveItemsTask::onStatusDone(KJob *job)
246 {
247     if (job->error()) {
248         qCWarning(IMAPRESOURCE_LOG) << job->errorString();
249         cancelTask(job->errorString());
250         return;
251     }
252 
253     auto status = qobject_cast<KIMAP::StatusJob *>(job);
254     const QList<QPair<QByteArray, qint64>> results = status->status();
255     for (const auto &val : results) {
256         if (val.first == "UIDNEXT") {
257             m_nextUid = val.second;
258             break;
259         }
260     }
261 
262     prepareRetrieval();
263 }
264 
prepareRetrieval()265 void RetrieveItemsTask::prepareRetrieval()
266 {
267     // Handle invalid UIDNEXT in case even STATUS is not able to retrieve it
268     if (m_nextUid < 0) {
269         qCWarning(IMAPRESOURCE_LOG) << "Server bug: Your IMAP Server delivered an invalid UIDNEXT value.";
270         m_nextUid = 0;
271     }
272 
273     // The select job retrieves highestmodseq whenever it's available, but in case of no CONDSTORE support we ignore it
274     if (!serverSupportsCondstore()) {
275         m_localHighestModSeq = 0;
276     }
277 
278     Akonadi::Collection col = collection();
279     bool modifyNeeded = false;
280 
281     // Get the current uid validity value and store it
282     int oldUidValidity = 0;
283     if (!col.hasAttribute("uidvalidity")) {
284         auto currentUidValidity = new UidValidityAttribute(m_uidValidity);
285         col.addAttribute(currentUidValidity);
286         modifyNeeded = true;
287     } else {
288         UidValidityAttribute *currentUidValidity = static_cast<UidValidityAttribute *>(col.attribute("uidvalidity"));
289         oldUidValidity = currentUidValidity->uidValidity();
290         if (oldUidValidity != m_uidValidity) {
291             currentUidValidity->setUidValidity(m_uidValidity);
292             modifyNeeded = true;
293         }
294     }
295 
296     // Get the current uid next value and store it
297     int oldNextUid = 0;
298     if (m_nextUid > 0) { // this can fail with faulty servers that don't deliver uidnext
299         if (auto currentNextUid = col.attribute<UidNextAttribute>()) {
300             oldNextUid = currentNextUid->uidNext();
301             if (oldNextUid != m_nextUid) {
302                 currentNextUid->setUidNext(m_nextUid);
303                 modifyNeeded = true;
304             }
305         } else {
306             col.attribute<UidNextAttribute>(Akonadi::Collection::AddIfMissing)->setUidNext(m_nextUid);
307             modifyNeeded = true;
308         }
309     }
310 
311     // Store the mailbox flags
312     if (!col.hasAttribute("collectionflags")) {
313         auto flagsAttribute = new Akonadi::CollectionFlagsAttribute(m_flags);
314         col.addAttribute(flagsAttribute);
315         modifyNeeded = true;
316     } else {
317         Akonadi::CollectionFlagsAttribute *flagsAttribute = static_cast<Akonadi::CollectionFlagsAttribute *>(col.attribute("collectionflags"));
318         const QList<QByteArray> oldFlags = flagsAttribute->flags();
319         if (oldFlags != m_flags) {
320             flagsAttribute->setFlags(m_flags);
321             modifyNeeded = true;
322         }
323     }
324 
325     qint64 oldHighestModSeq = 0;
326     if (serverSupportsCondstore() && m_highestModSeq > 0) {
327         if (!col.hasAttribute("highestmodseq")) {
328             auto attr = new HighestModSeqAttribute(m_highestModSeq);
329             col.addAttribute(attr);
330             modifyNeeded = true;
331         } else {
332             auto attr = col.attribute<HighestModSeqAttribute>();
333             if (attr->highestModSequence() < m_highestModSeq) {
334                 oldHighestModSeq = attr->highestModSequence();
335                 attr->setHighestModSeq(m_highestModSeq);
336                 modifyNeeded = true;
337             } else if (attr->highestModSequence() == m_highestModSeq) {
338                 oldHighestModSeq = attr->highestModSequence();
339             } else if (attr->highestModSequence() > m_highestModSeq) {
340                 // This situation should not happen. If it does, update the highestModSeq
341                 // attribute, but rather do a full sync
342                 attr->setHighestModSeq(m_highestModSeq);
343                 modifyNeeded = true;
344             }
345         }
346     }
347     m_localHighestModSeq = oldHighestModSeq;
348 
349     if (modifyNeeded) {
350         m_modifiedCollection = col;
351     }
352 
353     KIMAP::FetchJob::FetchScope scope;
354     scope.parts.clear();
355     scope.mode = KIMAP::FetchJob::FetchScope::FullHeaders;
356 
357     if (col.cachePolicy().localParts().contains(QLatin1String(Akonadi::MessagePart::Body))) {
358         scope.mode = KIMAP::FetchJob::FetchScope::Full;
359     }
360 
361     const qint64 realMessageCount = col.statistics().count();
362 
363     qCDebug(IMAPRESOURCE_LOG) << "Starting message retrieval. Elapsed(ms): " << m_time.elapsed();
364     qCDebug(IMAPRESOURCE_LOG) << "UidValidity: " << m_uidValidity << "Local UidValidity: " << oldUidValidity;
365     qCDebug(IMAPRESOURCE_LOG) << "MessageCount: " << m_messageCount << "Local message count: " << realMessageCount;
366     qCDebug(IMAPRESOURCE_LOG) << "UidNext: " << m_nextUid << "Local UidNext: " << oldNextUid;
367     qCDebug(IMAPRESOURCE_LOG) << "HighestModSeq: " << m_highestModSeq << "Local HighestModSeq: " << oldHighestModSeq;
368 
369     /*
370      * A synchronization has 3 mandatory steps:
371      * * If uidvalidity changed the local cache must be invalidated
372      * * New messages can be fetched using uidNext and the last known fetched uid
373      * * flag changes and removals can be detected by listing all messages that weren't part of the previous step
374      *
375      * Everything else is optimizations.
376      *
377      * TODO: Note that the local message count can be larger than the remote message count although no messages
378      * have been deleted remotely, if we locally have messages that were not yet uploaded.
379      * We cannot differentiate that from remotely removed messages, so we have to do a full flag
380      * listing in that case. This can be optimized once we support QRESYNC and therefore have a way
381      * to determine whether messages have been removed.
382      */
383 
384     if (m_messageCount == 0) {
385         // Shortcut:
386         // If no messages are present on the server, clear local cash and finish
387         m_incremental = false;
388         if (realMessageCount > 0) {
389             qCDebug(IMAPRESOURCE_LOG) << "No messages present so we are done, deleting local messages.";
390             itemsRetrieved(Akonadi::Item::List());
391         } else {
392             qCDebug(IMAPRESOURCE_LOG) << "No messages present so we are done";
393         }
394         taskComplete();
395     } else if (oldUidValidity != m_uidValidity || m_nextUid <= 0) {
396         // If uidvalidity has changed our local cache is worthless and has to be refetched completely
397         if (oldUidValidity != 0 && oldUidValidity != m_uidValidity) {
398             qCDebug(IMAPRESOURCE_LOG) << "UIDVALIDITY check failed (" << oldUidValidity << "|" << m_uidValidity << ")";
399         }
400         if (m_nextUid <= 0) {
401             qCDebug(IMAPRESOURCE_LOG) << "Invalid UIDNEXT";
402         }
403         qCDebug(IMAPRESOURCE_LOG) << "Fetching complete mailbox " << m_mailBox;
404         setTotalItems(m_messageCount);
405         retrieveItems(KIMAP::ImapSet(1, m_nextUid), scope, false, true);
406     } else if (m_nextUid <= 0) {
407         // This is a compatibility codepath for Courier IMAP. It probably introduces problems, but at least it syncs.
408         // Since we don't have uidnext available, we simply use the messagecount. This will miss simultaneously added/removed messages.
409         // qCDebug(IMAPRESOURCE_LOG) << "Running courier imap compatibility codepath";
410         if (m_messageCount > realMessageCount) {
411             // Get new messages
412             retrieveItems(KIMAP::ImapSet(realMessageCount + 1, m_messageCount), scope, false, false);
413         } else if (m_messageCount == realMessageCount) {
414             m_uidBasedFetch = false;
415             m_incremental = true;
416             setTotalItems(m_messageCount);
417             listFlagsForImapSet(KIMAP::ImapSet(1, m_messageCount));
418         } else {
419             m_uidBasedFetch = false;
420             m_incremental = false;
421             setTotalItems(m_messageCount);
422             listFlagsForImapSet(KIMAP::ImapSet(1, m_messageCount));
423         }
424     } else if (!m_messageUidsMissingBody.isEmpty()) {
425         // fetch missing uids
426         m_fetchedMissingBodies = 0;
427         setTotalItems(m_messageUidsMissingBody.size());
428         KIMAP::ImapSet imapSet;
429         imapSet.add(m_messageUidsMissingBody);
430         retrieveItems(imapSet, scope, true, true);
431     } else if (m_nextUid > oldNextUid && ((realMessageCount + m_nextUid - oldNextUid) == m_messageCount) && realMessageCount > 0) {
432         // Optimization:
433         // New messages are available, but we know no messages have been removed.
434         // Fetch new messages, and then check for changed flags and removed messages
435         // We can make an incremental update and use modseq.
436         qCDebug(IMAPRESOURCE_LOG) << "Incrementally fetching new messages: UidNext: " << m_nextUid << " Old UidNext: " << oldNextUid << " message count "
437                                   << m_messageCount << realMessageCount;
438         setTotalItems(qMax(1ll, m_messageCount - realMessageCount));
439         m_flagsChanged = !(m_highestModSeq == oldHighestModSeq);
440         retrieveItems(KIMAP::ImapSet(qMax(1, oldNextUid), m_nextUid), scope, true, true);
441     } else if (m_nextUid > oldNextUid && m_messageCount > (realMessageCount + m_nextUid - oldNextUid) && realMessageCount > 0) {
442         // Error recovery:
443         // New messages are available, but not enough to justify the difference between the local and remote message count.
444         // This can be triggered if we i.e. clear the local cache, but the keep the annotations.
445         // If we didn't catch this case, we end up inserting flags only for every missing message.
446         qCWarning(IMAPRESOURCE_LOG) << m_mailBox << ": detected inconsistency in local cache, we're missing some messages. Server: " << m_messageCount
447                                     << " Local: " << realMessageCount;
448         qCWarning(IMAPRESOURCE_LOG) << m_mailBox << ": refetching complete mailbox";
449         setTotalItems(m_messageCount);
450         retrieveItems(KIMAP::ImapSet(1, m_nextUid), scope, false, true);
451     } else if (m_nextUid > oldNextUid) {
452         // New messages are available. Fetch new messages, and then check for changed flags and removed messages
453         qCDebug(IMAPRESOURCE_LOG) << "Fetching new messages: UidNext: " << m_nextUid << " Old UidNext: " << oldNextUid;
454         setTotalItems(m_messageCount);
455         retrieveItems(KIMAP::ImapSet(qMax(1, oldNextUid), m_nextUid), scope, false, true);
456     } else if (m_messageCount == realMessageCount && oldNextUid == m_nextUid) {
457         // Optimization:
458         // We know no messages were added or removed (if the message count and uidnext is still the same)
459         // We only check the flags incrementally and can make use of modseq
460         m_uidBasedFetch = true;
461         m_incremental = true;
462         m_flagsChanged = !(m_highestModSeq == oldHighestModSeq);
463         // Workaround: If the server doesn't support CONDSTORE we would end up syncing all flags during every sync.
464         // Instead we only sync flags when new messages are available or removed and skip this step.
465         // WARNING: This sacrifices consistency as we will not detect flag changes until a new message enters the mailbox.
466         if (m_incremental && !serverSupportsCondstore()) {
467             qCDebug(IMAPRESOURCE_LOG) << "Avoiding flag sync due to missing CONDSTORE support";
468             taskComplete();
469             return;
470         }
471         setTotalItems(m_messageCount);
472         listFlagsForImapSet(KIMAP::ImapSet(1, m_nextUid));
473     } else if (m_messageCount > realMessageCount) {
474         // Error recovery:
475         // We didn't detect any new messages based on the uid, but according to the message count there are new ones.
476         // Our local cache is invalid and has to be refetched.
477         qCWarning(IMAPRESOURCE_LOG) << m_mailBox << ": detected inconsistency in local cache, we're missing some messages. Server: " << m_messageCount
478                                     << " Local: " << realMessageCount;
479         qCWarning(IMAPRESOURCE_LOG) << m_mailBox << ": refetching complete mailbox";
480         setTotalItems(m_messageCount);
481         retrieveItems(KIMAP::ImapSet(1, m_nextUid), scope, false, true);
482     } else {
483         // Shortcut:
484         // No new messages are available. Directly check for changed flags and removed messages.
485         m_uidBasedFetch = true;
486         m_incremental = false;
487         setTotalItems(m_messageCount);
488         listFlagsForImapSet(KIMAP::ImapSet(1, m_nextUid));
489     }
490 }
491 
retrieveItems(const KIMAP::ImapSet & set,const KIMAP::FetchJob::FetchScope & scope,bool incremental,bool uidBased)492 void RetrieveItemsTask::retrieveItems(const KIMAP::ImapSet &set, const KIMAP::FetchJob::FetchScope &scope, bool incremental, bool uidBased)
493 {
494     Q_ASSERT(set.intervals().size() == 1);
495 
496     m_incremental = incremental;
497     m_uidBasedFetch = uidBased;
498 
499     m_batchFetcher = createBatchFetcher(resourceState()->messageHelper(), set, scope, batchSize(), m_session);
500     m_batchFetcher->setUidBased(m_uidBasedFetch);
501     if (m_uidBasedFetch && set.intervals().size() == 1) {
502         m_batchFetcher->setSearchUids(set.intervals().front());
503     }
504     m_batchFetcher->setProperty("alreadyFetched", set.intervals().at(0).begin());
505     connect(m_batchFetcher, &BatchFetcher::itemsRetrieved, this, &RetrieveItemsTask::onItemsRetrieved);
506     connect(m_batchFetcher, &KJob::result, this, &RetrieveItemsTask::onRetrievalDone);
507     m_batchFetcher->start();
508 }
509 
onReadyForNextBatch(int size)510 void RetrieveItemsTask::onReadyForNextBatch(int size)
511 {
512     Q_UNUSED(size)
513     if (m_batchFetcher) {
514         m_batchFetcher->fetchNextBatch();
515     }
516 }
517 
onItemsRetrieved(const Akonadi::Item::List & addedItems)518 void RetrieveItemsTask::onItemsRetrieved(const Akonadi::Item::List &addedItems)
519 {
520     if (m_incremental) {
521         itemsRetrievedIncremental(addedItems, Akonadi::Item::List());
522     } else {
523         itemsRetrieved(addedItems);
524     }
525 
526     // m_fetchedMissingBodies is -1 if we fetch for other reason, but missing bodies
527     if (m_fetchedMissingBodies != -1) {
528         const QString mailBox = mailBoxForCollection(collection());
529         m_fetchedMissingBodies += addedItems.count();
530         Q_EMIT status(Akonadi::AgentBase::Running,
531                       i18nc("@info:status", "Fetching missing mail bodies in %3: %1/%2", m_fetchedMissingBodies, m_messageUidsMissingBody.count(), mailBox));
532     }
533 }
534 
onRetrievalDone(KJob * job)535 void RetrieveItemsTask::onRetrievalDone(KJob *job)
536 {
537     m_batchFetcher = nullptr;
538     if (job->error()) {
539         qCWarning(IMAPRESOURCE_LOG) << job->errorString();
540         cancelTask(job->errorString());
541         m_fetchedMissingBodies = -1;
542         return;
543     }
544 
545     // This is the lowest sequence number that we just fetched.
546     const auto alreadyFetchedBegin = job->property("alreadyFetched").value<KIMAP::ImapSet::Id>();
547 
548     // If this is the first fetch of a folder, skip getting flags, we
549     // already have them all from the previous full fetch. This is not
550     // just an optimization, as incremental retrieval assumes nothing
551     // will be listed twice.
552     if (m_fetchedMissingBodies != -1 || alreadyFetchedBegin <= 1) {
553         taskComplete();
554         return;
555     }
556 
557     // Fetch flags of all items that were not fetched by the fetchJob. After
558     // that /all/ items in the folder are synced.
559     listFlagsForImapSet(KIMAP::ImapSet(1, alreadyFetchedBegin - 1));
560 }
561 
listFlagsForImapSet(const KIMAP::ImapSet & set)562 void RetrieveItemsTask::listFlagsForImapSet(const KIMAP::ImapSet &set)
563 {
564     qCDebug(IMAPRESOURCE_LOG) << "Listing flags " << set.intervals().at(0).begin() << set.intervals().at(0).end();
565     qCDebug(IMAPRESOURCE_LOG) << "Starting flag retrieval. Elapsed(ms): " << m_time.elapsed();
566 
567     KIMAP::FetchJob::FetchScope scope;
568     scope.parts.clear();
569     scope.mode = KIMAP::FetchJob::FetchScope::Flags;
570     // Only use changeSince when doing incremental listings,
571     // otherwise we would overwrite our local data with an incomplete dataset
572     if (m_incremental && serverSupportsCondstore()) {
573         scope.changedSince = m_localHighestModSeq;
574         if (!m_flagsChanged) {
575             qCDebug(IMAPRESOURCE_LOG) << "No flag changes.";
576             taskComplete();
577             return;
578         }
579     }
580 
581     m_batchFetcher = createBatchFetcher(resourceState()->messageHelper(), set, scope, 10 * batchSize(), m_session);
582     m_batchFetcher->setUidBased(m_uidBasedFetch);
583     if (m_uidBasedFetch && scope.changedSince == 0 && set.intervals().size() == 1) {
584         m_batchFetcher->setSearchUids(set.intervals().front());
585     }
586     connect(m_batchFetcher, &BatchFetcher::itemsRetrieved, this, &RetrieveItemsTask::onItemsRetrieved);
587     connect(m_batchFetcher, &KJob::result, this, &RetrieveItemsTask::onFlagsFetchDone);
588     m_batchFetcher->start();
589 }
590 
onFlagsFetchDone(KJob * job)591 void RetrieveItemsTask::onFlagsFetchDone(KJob *job)
592 {
593     m_batchFetcher = nullptr;
594     if (job->error()) {
595         qCWarning(IMAPRESOURCE_LOG) << job->errorString();
596         cancelTask(job->errorString());
597     } else {
598         taskComplete();
599     }
600 }
601 
taskComplete()602 void RetrieveItemsTask::taskComplete()
603 {
604     if (m_modifiedCollection.isValid()) {
605         qCDebug(IMAPRESOURCE_LOG) << "Applying collection changes";
606         applyCollectionChanges(m_modifiedCollection);
607     }
608     if (m_incremental) {
609         // Calling itemsRetrievalDone() before previous call to itemsRetrievedIncremental()
610         // behaves like if we called itemsRetrieved(Items::List()), so make sure
611         // Akonadi knows we did incremental fetch that came up with no changes
612         itemsRetrievedIncremental(Akonadi::Item::List(), Akonadi::Item::List());
613     }
614     qCDebug(IMAPRESOURCE_LOG) << "Retrieval complete. Elapsed(ms): " << m_time.elapsed();
615     itemsRetrievalDone();
616 }
617