1 /* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "seq-range-array.h"
5 #include "ioloop.h"
6 #include "array.h"
7 #include "index-mailbox-size.h"
8 #include "index-sync-private.h"
9 #include "mailbox-recent-flags.h"
10
11 struct index_storage_list_index_record {
12 uint32_t size;
13 uint32_t mtime;
14 };
15
index_storage_get_sync_flags(struct mailbox * box)16 enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box)
17 {
18 enum mail_index_sync_flags sync_flags = 0;
19
20 if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0)
21 sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
22 if (box->deleting) {
23 sync_flags |= box->delete_sync_check ?
24 MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX :
25 MAIL_INDEX_SYNC_FLAG_DELETING_INDEX;
26 }
27 return sync_flags;
28 }
29
index_mailbox_want_full_sync(struct mailbox * box,enum mailbox_sync_flags flags)30 bool index_mailbox_want_full_sync(struct mailbox *box,
31 enum mailbox_sync_flags flags)
32 {
33 struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
34
35 if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 &&
36 ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL)
37 return FALSE;
38
39 if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 &&
40 (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) {
41 /* lib-lda is syncing the mailbox after saving a mail.
42 it only wants to find the new mail for potentially copying
43 to other mailboxes. that's mainly an optimization, and since
44 the mail was most likely already added to index we don't
45 need to do a full sync to find it. the main benefit here is
46 to avoid a very costly sync with a large Maildir/new/ */
47 return FALSE;
48 }
49
50 if (box->to_notify != NULL)
51 timeout_reset(box->to_notify);
52 ibox->sync_last_check = ioloop_time;
53 return TRUE;
54 }
55
index_view_sync_recs_get(struct index_mailbox_sync_context * ctx)56 static void index_view_sync_recs_get(struct index_mailbox_sync_context *ctx)
57 {
58 struct mail_index_view_sync_rec sync_rec;
59 uint32_t seq1, seq2;
60
61 i_array_init(&ctx->flag_updates, 128);
62 i_array_init(&ctx->hidden_updates, 32);
63 while (mail_index_view_sync_next(ctx->sync_ctx, &sync_rec)) {
64 switch (sync_rec.type) {
65 case MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ:
66 case MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS:
67 if (!mail_index_lookup_seq_range(ctx->ctx.box->view,
68 sync_rec.uid1,
69 sync_rec.uid2,
70 &seq1, &seq2))
71 break;
72
73 if (!sync_rec.hidden &&
74 sync_rec.type == MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) {
75 seq_range_array_add_range(&ctx->flag_updates,
76 seq1, seq2);
77 } else {
78 seq_range_array_add_range(&ctx->hidden_updates,
79 seq1, seq2);
80 }
81 break;
82 }
83 }
84 }
85
86 static void
index_view_sync_cleanup_updates(struct index_mailbox_sync_context * ctx)87 index_view_sync_cleanup_updates(struct index_mailbox_sync_context *ctx)
88 {
89 /* remove expunged messages from flag updates */
90 if (ctx->expunges != NULL) {
91 seq_range_array_remove_seq_range(&ctx->flag_updates,
92 ctx->expunges);
93 seq_range_array_remove_seq_range(&ctx->hidden_updates,
94 ctx->expunges);
95 }
96 /* remove flag updates from hidden updates */
97 seq_range_array_remove_seq_range(&ctx->hidden_updates,
98 &ctx->flag_updates);
99 }
100
101 struct mailbox_sync_context *
index_mailbox_sync_init(struct mailbox * box,enum mailbox_sync_flags flags,bool failed)102 index_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags,
103 bool failed)
104 {
105 struct index_mailbox_sync_context *ctx;
106 struct index_mailbox_sync_pvt_context *pvt_ctx;
107 enum mail_index_view_sync_flags sync_flags = 0;
108
109 ctx = i_new(struct index_mailbox_sync_context, 1);
110 ctx->ctx.box = box;
111 ctx->ctx.flags = flags;
112
113 if (failed) {
114 ctx->failed = TRUE;
115 return &ctx->ctx;
116 }
117
118 if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0)
119 sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES;
120
121 if ((flags & MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
122 sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT;
123 ctx->messages_count = 0;
124 } else {
125 ctx->messages_count =
126 mail_index_view_get_messages_count(box->view);
127 }
128
129 if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0) {
130 /* we most likely did a fast sync. refresh the index anyway in
131 case there were some new changes. */
132 (void)mail_index_refresh(box->index);
133 }
134 ctx->sync_ctx = mail_index_view_sync_begin(box->view, sync_flags);
135 if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0) {
136 mail_index_view_sync_get_expunges(ctx->sync_ctx,
137 &ctx->expunges);
138 ctx->expunge_pos = array_count(ctx->expunges);
139 }
140 index_view_sync_recs_get(ctx);
141 index_sync_search_results_expunge(ctx);
142
143 /* sync private index if needed. it doesn't use box->view, so it
144 doesn't matter if it's called at _sync_init() or _sync_deinit().
145 however we also need to know if any private flags have changed
146 since last sync, so we need to call it before _sync_next() calls. */
147 if (index_mailbox_sync_pvt_init(box, FALSE, sync_flags, &pvt_ctx) > 0) {
148 (void)index_mailbox_sync_pvt_view(pvt_ctx, &ctx->flag_updates,
149 &ctx->hidden_updates);
150 index_mailbox_sync_pvt_deinit(&pvt_ctx);
151
152 }
153 index_view_sync_cleanup_updates(ctx);
154 return &ctx->ctx;
155 }
156
157 static bool
index_mailbox_sync_next_expunge(struct index_mailbox_sync_context * ctx,struct mailbox_sync_rec * sync_rec_r)158 index_mailbox_sync_next_expunge(struct index_mailbox_sync_context *ctx,
159 struct mailbox_sync_rec *sync_rec_r)
160 {
161 const struct seq_range *range;
162
163 if (ctx->expunge_pos == 0)
164 return FALSE;
165
166 /* expunges is a sorted array of sequences. it's easiest for
167 us to print them from end to beginning. */
168 ctx->expunge_pos--;
169 range = array_idx(ctx->expunges, ctx->expunge_pos);
170 i_assert(range->seq2 <= ctx->messages_count);
171
172 mailbox_recent_flags_expunge_seqs(ctx->ctx.box, range->seq1, range->seq2);
173 ctx->messages_count -= range->seq2 - range->seq1 + 1;
174
175 sync_rec_r->seq1 = range->seq1;
176 sync_rec_r->seq2 = range->seq2;
177 sync_rec_r->type = MAILBOX_SYNC_TYPE_EXPUNGE;
178 return TRUE;
179 }
180
index_mailbox_sync_next(struct mailbox_sync_context * _ctx,struct mailbox_sync_rec * sync_rec_r)181 bool index_mailbox_sync_next(struct mailbox_sync_context *_ctx,
182 struct mailbox_sync_rec *sync_rec_r)
183 {
184 struct index_mailbox_sync_context *ctx =
185 (struct index_mailbox_sync_context *)_ctx;
186 const struct seq_range *range;
187 unsigned int count;
188
189 if (ctx->failed)
190 return FALSE;
191
192 range = array_get(&ctx->flag_updates, &count);
193 if (ctx->flag_update_idx < count) {
194 sync_rec_r->type = MAILBOX_SYNC_TYPE_FLAGS;
195 sync_rec_r->seq1 = range[ctx->flag_update_idx].seq1;
196 sync_rec_r->seq2 = range[ctx->flag_update_idx].seq2;
197 ctx->flag_update_idx++;
198 return TRUE;
199 }
200 if ((_ctx->box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0) {
201 /* hidden flag changes' MODSEQs still need to be returned */
202 range = array_get(&ctx->hidden_updates, &count);
203 if (ctx->hidden_update_idx < count) {
204 sync_rec_r->type = MAILBOX_SYNC_TYPE_MODSEQ;
205 sync_rec_r->seq1 = range[ctx->hidden_update_idx].seq1;
206 sync_rec_r->seq2 = range[ctx->hidden_update_idx].seq2;
207 ctx->hidden_update_idx++;
208 return TRUE;
209 }
210 }
211
212 return index_mailbox_sync_next_expunge(ctx, sync_rec_r);
213 }
214
215 static void
index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context * ctx)216 index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx)
217 {
218 struct mailbox *box = ctx->ctx.box;
219 struct mail_index_view *view = ctx->ctx.box->view;
220 const struct mail_index_header *hdr;
221 uint32_t seq, start_uid, uid;
222
223 if (!array_is_created(&box->recent_flags))
224 return;
225
226 /* expunges array contained expunges for the messages that were already
227 visible in this view, but append+expunge would be invisible.
228 recent_flags may however contain the append UID, so we'll have to
229 remove it separately */
230 hdr = mail_index_get_header(view);
231 if (ctx->messages_count == 0)
232 uid = 0;
233 else if (ctx->messages_count <= hdr->messages_count)
234 mail_index_lookup_uid(view, ctx->messages_count, &uid);
235 else {
236 i_assert(mail_index_view_is_inconsistent(view));
237 return;
238 }
239
240 for (seq = ctx->messages_count + 1; seq <= hdr->messages_count; seq++) {
241 start_uid = uid;
242 mail_index_lookup_uid(view, seq, &uid);
243 if (start_uid + 1 > uid - 1)
244 continue;
245
246 box->recent_flags_count -=
247 seq_range_array_remove_range(&box->recent_flags,
248 start_uid + 1, uid - 1);
249 }
250
251 if (uid + 1 < hdr->next_uid) {
252 box->recent_flags_count -=
253 seq_range_array_remove_range(&box->recent_flags,
254 uid + 1,
255 hdr->next_uid - 1);
256 }
257 #ifdef DEBUG
258 if (!mail_index_view_is_inconsistent(view)) {
259 const struct seq_range *range;
260 unsigned int i, count;
261
262 range = array_get(&box->recent_flags, &count);
263 for (i = 0; i < count; i++) {
264 for (uid = range[i].seq1; uid <= range[i].seq2; uid++) {
265 if (uid >= hdr->next_uid)
266 break;
267 (void)mail_index_lookup_seq(view, uid, &seq);
268 i_assert(seq != 0);
269 }
270 }
271 }
272 #endif
273 }
274
index_sync_update_recent_count(struct mailbox * box)275 void index_sync_update_recent_count(struct mailbox *box)
276 {
277 struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
278 const struct mail_index_header *hdr;
279 uint32_t seq1, seq2;
280
281 hdr = mail_index_get_header(box->view);
282 if (hdr->first_recent_uid < ibox->recent_flags_prev_first_recent_uid) {
283 mailbox_set_critical(box,
284 "first_recent_uid unexpectedly shrank: %u -> %u",
285 ibox->recent_flags_prev_first_recent_uid,
286 hdr->first_recent_uid);
287 mailbox_recent_flags_reset(box);
288 }
289
290 if (hdr->first_recent_uid > box->recent_flags_prev_uid ||
291 hdr->next_uid > ibox->recent_flags_last_check_nextuid) {
292 ibox->recent_flags_prev_first_recent_uid = hdr->first_recent_uid;
293 ibox->recent_flags_last_check_nextuid = hdr->next_uid;
294 if (mail_index_lookup_seq_range(box->view,
295 hdr->first_recent_uid,
296 hdr->next_uid,
297 &seq1, &seq2)) {
298 mailbox_recent_flags_set_seqs(box, box->view,
299 seq1, seq2);
300 }
301 }
302 }
303
index_mailbox_sync_free(struct index_mailbox_sync_context * ctx)304 static void index_mailbox_sync_free(struct index_mailbox_sync_context *ctx)
305 {
306 if (array_is_created(&ctx->flag_updates))
307 array_free(&ctx->flag_updates);
308 if (array_is_created(&ctx->hidden_updates))
309 array_free(&ctx->hidden_updates);
310 if (array_is_created(&ctx->all_flag_update_uids))
311 array_free(&ctx->all_flag_update_uids);
312 i_free(ctx);
313 }
314
index_mailbox_sync_deinit(struct mailbox_sync_context * _ctx,struct mailbox_sync_status * status_r)315 int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
316 struct mailbox_sync_status *status_r)
317 {
318 struct index_mailbox_sync_context *ctx =
319 (struct index_mailbox_sync_context *)_ctx;
320 struct mailbox_sync_rec sync_rec;
321 bool delayed_expunges = FALSE;
322 int ret = ctx->failed ? -1 : 0;
323
324 /* finish handling expunges, so we don't break when updating
325 recent flags */
326 while (index_mailbox_sync_next_expunge(ctx, &sync_rec)) ;
327
328 /* convert sequences to uids before syncing view */
329 index_sync_search_results_uidify(ctx);
330
331 if (ctx->sync_ctx != NULL) {
332 if (mail_index_view_sync_commit(&ctx->sync_ctx,
333 &delayed_expunges) < 0) {
334 mailbox_set_index_error(_ctx->box);
335 ret = -1;
336 }
337 }
338 if (ret < 0) {
339 index_mailbox_sync_free(ctx);
340 return -1;
341 }
342 index_mailbox_expunge_unseen_recent(ctx);
343
344 if ((_ctx->box->flags & MAILBOX_FLAG_DROP_RECENT) == 0 &&
345 _ctx->box->opened) {
346 /* mailbox syncing didn't necessarily update our recent state */
347 index_sync_update_recent_count(_ctx->box);
348 }
349
350 if (status_r != NULL)
351 status_r->sync_delayed_expunges = delayed_expunges;
352
353 /* update search results after private index is updated */
354 index_sync_search_results_update(ctx);
355 /* update vsize header if wanted */
356 index_mailbox_vsize_update_appends(_ctx->box);
357
358 if (ret == 0 && mail_index_view_is_inconsistent(_ctx->box->view)) {
359 /* we probably had MAILBOX_SYNC_FLAG_FIX_INCONSISTENT set,
360 but the view got broken in the middle. FIXME: We could
361 attempt to fix it automatically. In any case now the view
362 isn't usable and we can't return success. */
363 mailbox_set_index_error(_ctx->box);
364 ret = -1;
365 }
366
367 index_mailbox_sync_free(ctx);
368 return ret;
369 }
370
index_keyword_array_cmp(const ARRAY_TYPE (keyword_indexes)* k1,const ARRAY_TYPE (keyword_indexes)* k2)371 bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1,
372 const ARRAY_TYPE(keyword_indexes) *k2)
373 {
374 const unsigned int *idx1, *idx2;
375 unsigned int i, j, count1, count2;
376
377 if (!array_is_created(k1))
378 return !array_is_created(k2) || array_count(k2) == 0;
379 if (!array_is_created(k2))
380 return array_count(k1) == 0;
381
382 /* The arrays may not be sorted, but they usually are. Optimize for
383 the assumption that they are */
384 idx1 = array_get(k1, &count1);
385 idx2 = array_get(k2, &count2);
386
387 if (count1 != count2)
388 return FALSE;
389
390 for (i = 0; i < count1; i++) {
391 if (idx1[i] != idx2[i]) {
392 /* not found / unsorted array. check. */
393 for (j = 0; j < count1; j++) {
394 if (idx1[i] == idx2[j])
395 break;
396 }
397 if (j == count1)
398 return FALSE;
399 }
400 }
401 return TRUE;
402 }
403
index_sync_type_convert(enum mail_index_sync_type type)404 enum mailbox_sync_type index_sync_type_convert(enum mail_index_sync_type type)
405 {
406 enum mailbox_sync_type ret = 0;
407
408 if ((type & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0)
409 ret |= MAILBOX_SYNC_TYPE_EXPUNGE;
410 if ((type & (MAIL_INDEX_SYNC_TYPE_FLAGS |
411 MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD |
412 MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)) != 0)
413 ret |= MAILBOX_SYNC_TYPE_FLAGS;
414 return ret;
415 }
416
417 static uint32_t
index_list_get_ext_id(struct mailbox * box,struct mail_index_view * view)418 index_list_get_ext_id(struct mailbox *box, struct mail_index_view *view)
419 {
420 struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
421
422 if (ibox->list_index_sync_ext_id == (uint32_t)-1) {
423 ibox->list_index_sync_ext_id =
424 mail_index_ext_register(mail_index_view_get_index(view),
425 "index sync", 0,
426 sizeof(struct index_storage_list_index_record),
427 sizeof(uint32_t));
428 }
429 return ibox->list_index_sync_ext_id;
430 }
431
432 enum index_storage_list_change
index_storage_list_index_has_changed_full(struct mailbox * box,struct mail_index_view * list_view,uint32_t seq)433 index_storage_list_index_has_changed_full(struct mailbox *box,
434 struct mail_index_view *list_view,
435 uint32_t seq)
436 {
437 const struct index_storage_list_index_record *rec;
438 const void *data;
439 const char *dir, *path;
440 struct stat st;
441 uint32_t ext_id;
442 bool expunged;
443 int ret;
444
445 if (mail_index_is_in_memory(mail_index_view_get_index(list_view)))
446 return INDEX_STORAGE_LIST_CHANGE_INMEMORY;
447
448 ext_id = index_list_get_ext_id(box, list_view);
449 mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
450 rec = data;
451
452 if (rec == NULL || expunged || rec->size == 0 || rec->mtime == 0) {
453 /* doesn't exist / not synced */
454 return INDEX_STORAGE_LIST_CHANGE_NORECORD;
455 }
456 if (box->storage->set->mailbox_list_index_very_dirty_syncs)
457 return INDEX_STORAGE_LIST_CHANGE_NONE;
458
459 ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir);
460 if (ret < 0)
461 return INDEX_STORAGE_LIST_CHANGE_ERROR;
462 i_assert(ret > 0);
463
464 path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL);
465 if (stat(path, &st) < 0) {
466 if (errno == ENOENT)
467 return INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS;
468 mailbox_set_critical(box, "stat(%s) failed: %m", path);
469 return INDEX_STORAGE_LIST_CHANGE_ERROR;
470 }
471 if (rec->size != (st.st_size & 0xffffffffU))
472 return INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED;
473 if (rec->mtime != (st.st_mtime & 0xffffffffU))
474 return INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED;
475 return INDEX_STORAGE_LIST_CHANGE_NONE;
476 }
477
index_storage_list_index_has_changed(struct mailbox * box,struct mail_index_view * list_view,uint32_t seq,bool quick ATTR_UNUSED)478 int index_storage_list_index_has_changed(struct mailbox *box,
479 struct mail_index_view *list_view,
480 uint32_t seq, bool quick ATTR_UNUSED)
481 {
482 switch (index_storage_list_index_has_changed_full(box, list_view, seq)) {
483 case INDEX_STORAGE_LIST_CHANGE_ERROR:
484 return -1;
485 case INDEX_STORAGE_LIST_CHANGE_NONE:
486 return 0;
487 default:
488 return 1;
489 }
490 }
491
index_storage_list_index_update_sync(struct mailbox * box,struct mail_index_transaction * trans,uint32_t seq)492 void index_storage_list_index_update_sync(struct mailbox *box,
493 struct mail_index_transaction *trans,
494 uint32_t seq)
495 {
496 struct mail_index_view *list_view;
497 const struct index_storage_list_index_record *old_rec;
498 struct index_storage_list_index_record new_rec;
499 const void *data;
500 const char *dir, *path;
501 struct stat st;
502 uint32_t ext_id;
503 bool expunged;
504 int ret;
505
506 list_view = mail_index_transaction_get_view(trans);
507 if (mail_index_is_in_memory(mail_index_view_get_index(list_view)))
508 return;
509
510 /* get the current record */
511 ext_id = index_list_get_ext_id(box, list_view);
512 mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
513 if (expunged)
514 return;
515 old_rec = data;
516
517 ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir);
518 if (ret < 0)
519 return;
520 i_assert(ret > 0);
521
522 path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL);
523 if (stat(path, &st) < 0) {
524 mailbox_set_critical(box, "stat(%s) failed: %m", path);
525 return;
526 }
527
528 i_zero(&new_rec);
529 new_rec.size = st.st_size & 0xffffffffU;
530 new_rec.mtime = st.st_mtime & 0xffffffffU;
531
532 if (old_rec == NULL ||
533 memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0)
534 mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL);
535 }
536