1 /* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "array.h"
5 #include "mail-index-view-private.h"
6 #include "mail-index-sync-private.h"
7 #include "mail-index-transaction-private.h"
8 #include "mail-transaction-log-private.h"
9 #include "mail-cache-private.h"
10
11 #include <stdio.h>
12
13 struct mail_index_sync_ctx {
14 struct mail_index *index;
15 struct mail_index_view *view;
16 struct mail_index_transaction *sync_trans, *ext_trans;
17 struct mail_index_transaction_commit_result *sync_commit_result;
18 enum mail_index_sync_flags flags;
19 char *reason;
20
21 const struct mail_transaction_header *hdr;
22 const void *data;
23
24 ARRAY(struct mail_index_sync_list) sync_list;
25 uint32_t next_uid;
26
27 bool no_warning:1;
28 bool seen_external_expunges:1;
29 bool seen_nonexternal_transactions:1;
30 bool fully_synced:1;
31 };
32
mail_index_sync_add_expunge(struct mail_index_sync_ctx * ctx)33 static void mail_index_sync_add_expunge(struct mail_index_sync_ctx *ctx)
34 {
35 const struct mail_transaction_expunge *e = ctx->data;
36 size_t i, size = ctx->hdr->size / sizeof(*e);
37 uint32_t uid;
38
39 for (i = 0; i < size; i++) {
40 for (uid = e[i].uid1; uid <= e[i].uid2; uid++)
41 mail_index_expunge(ctx->sync_trans, uid);
42 }
43 }
44
mail_index_sync_add_expunge_guid(struct mail_index_sync_ctx * ctx)45 static void mail_index_sync_add_expunge_guid(struct mail_index_sync_ctx *ctx)
46 {
47 const struct mail_transaction_expunge_guid *e = ctx->data;
48 size_t i, size = ctx->hdr->size / sizeof(*e);
49
50 for (i = 0; i < size; i++) {
51 mail_index_expunge_guid(ctx->sync_trans, e[i].uid,
52 e[i].guid_128);
53 }
54 }
55
mail_index_sync_add_flag_update(struct mail_index_sync_ctx * ctx)56 static void mail_index_sync_add_flag_update(struct mail_index_sync_ctx *ctx)
57 {
58 const struct mail_transaction_flag_update *u = ctx->data;
59 size_t i, size = ctx->hdr->size / sizeof(*u);
60
61 for (i = 0; i < size; i++) {
62 if (u[i].add_flags != 0) {
63 mail_index_update_flags_range(ctx->sync_trans,
64 u[i].uid1, u[i].uid2,
65 MODIFY_ADD,
66 u[i].add_flags);
67 }
68 if (u[i].remove_flags != 0) {
69 mail_index_update_flags_range(ctx->sync_trans,
70 u[i].uid1, u[i].uid2,
71 MODIFY_REMOVE,
72 u[i].remove_flags);
73 }
74 }
75 }
76
mail_index_sync_add_keyword_update(struct mail_index_sync_ctx * ctx)77 static void mail_index_sync_add_keyword_update(struct mail_index_sync_ctx *ctx)
78 {
79 const struct mail_transaction_keyword_update *u = ctx->data;
80 const char *keyword_names[2];
81 struct mail_keywords *keywords;
82 const uint32_t *uids;
83 uint32_t uid;
84 size_t uidset_offset, i, size;
85
86 i_assert(u->name_size > 0);
87
88 uidset_offset = sizeof(*u) + u->name_size;
89 if ((uidset_offset % 4) != 0)
90 uidset_offset += 4 - (uidset_offset % 4);
91 uids = CONST_PTR_OFFSET(u, uidset_offset);
92
93 keyword_names[0] = t_strndup(u + 1, u->name_size);
94 keyword_names[1] = NULL;
95 keywords = mail_index_keywords_create(ctx->index, keyword_names);
96
97 size = (ctx->hdr->size - uidset_offset) / sizeof(uint32_t);
98 for (i = 0; i < size; i += 2) {
99 /* FIXME: mail_index_update_keywords_range() */
100 for (uid = uids[i]; uid <= uids[i+1]; uid++) {
101 mail_index_update_keywords(ctx->sync_trans, uid,
102 u->modify_type, keywords);
103 }
104 }
105
106 mail_index_keywords_unref(&keywords);
107 }
108
mail_index_sync_add_keyword_reset(struct mail_index_sync_ctx * ctx)109 static void mail_index_sync_add_keyword_reset(struct mail_index_sync_ctx *ctx)
110 {
111 const struct mail_transaction_keyword_reset *u = ctx->data;
112 size_t i, size = ctx->hdr->size / sizeof(*u);
113 struct mail_keywords *keywords;
114 uint32_t uid;
115
116 keywords = mail_index_keywords_create(ctx->index, NULL);
117 for (i = 0; i < size; i++) {
118 for (uid = u[i].uid1; uid <= u[i].uid2; uid++) {
119 mail_index_update_keywords(ctx->sync_trans, uid,
120 MODIFY_REPLACE, keywords);
121 }
122 }
123 mail_index_keywords_unref(&keywords);
124 }
125
mail_index_sync_add_transaction(struct mail_index_sync_ctx * ctx)126 static bool mail_index_sync_add_transaction(struct mail_index_sync_ctx *ctx)
127 {
128 switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
129 case MAIL_TRANSACTION_EXPUNGE:
130 mail_index_sync_add_expunge(ctx);
131 break;
132 case MAIL_TRANSACTION_EXPUNGE_GUID:
133 mail_index_sync_add_expunge_guid(ctx);
134 break;
135 case MAIL_TRANSACTION_FLAG_UPDATE:
136 mail_index_sync_add_flag_update(ctx);
137 break;
138 case MAIL_TRANSACTION_KEYWORD_UPDATE:
139 mail_index_sync_add_keyword_update(ctx);
140 break;
141 case MAIL_TRANSACTION_KEYWORD_RESET:
142 mail_index_sync_add_keyword_reset(ctx);
143 break;
144 default:
145 return FALSE;
146 }
147 return TRUE;
148 }
149
mail_index_sync_add_dirty_updates(struct mail_index_sync_ctx * ctx)150 static void mail_index_sync_add_dirty_updates(struct mail_index_sync_ctx *ctx)
151 {
152 struct mail_transaction_flag_update update;
153 const struct mail_index_record *rec;
154 uint32_t seq, messages_count;
155
156 i_zero(&update);
157
158 messages_count = mail_index_view_get_messages_count(ctx->view);
159 for (seq = 1; seq <= messages_count; seq++) {
160 rec = mail_index_lookup(ctx->view, seq);
161 if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0)
162 continue;
163
164 mail_index_update_flags(ctx->sync_trans, rec->uid,
165 MODIFY_REPLACE, rec->flags);
166 }
167 }
168
169 static int
mail_index_sync_read_and_sort(struct mail_index_sync_ctx * ctx)170 mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx)
171 {
172 struct mail_index_transaction *sync_trans = ctx->sync_trans;
173 struct mail_index_sync_list *synclist;
174 const struct mail_index_transaction_keyword_update *keyword_updates;
175 unsigned int i, keyword_count;
176 int ret;
177
178 if ((ctx->view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 &&
179 (ctx->flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 &&
180 (ctx->view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) {
181 /* show dirty flags as flag updates */
182 mail_index_sync_add_dirty_updates(ctx);
183 }
184
185 /* read all transactions from log into a transaction in memory.
186 skip the external ones, they're already synced to mailbox and
187 included in our view */
188 while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
189 &ctx->hdr,
190 &ctx->data)) > 0) {
191 if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
192 if ((ctx->hdr->type & (MAIL_TRANSACTION_EXPUNGE |
193 MAIL_TRANSACTION_EXPUNGE_GUID)) != 0)
194 ctx->seen_external_expunges = TRUE;
195 continue;
196 }
197
198 T_BEGIN {
199 if (mail_index_sync_add_transaction(ctx)) {
200 /* update tail_offset if needed */
201 ctx->seen_nonexternal_transactions = TRUE;
202 } else {
203 /* this is an internal change. we don't
204 necessarily need to update tail_offset, so
205 avoid the extra write caused by it. */
206 }
207 } T_END;
208 }
209
210 /* create an array containing all expunge, flag and keyword update
211 arrays so we can easily go through all of the changes. */
212 keyword_count = !array_is_created(&sync_trans->keyword_updates) ? 0 :
213 array_count(&sync_trans->keyword_updates);
214 i_array_init(&ctx->sync_list, keyword_count + 2);
215
216 if (array_is_created(&sync_trans->expunges)) {
217 mail_index_transaction_sort_expunges(sync_trans);
218 synclist = array_append_space(&ctx->sync_list);
219 synclist->array = (void *)&sync_trans->expunges;
220 }
221
222 if (array_is_created(&sync_trans->updates)) {
223 synclist = array_append_space(&ctx->sync_list);
224 synclist->array = (void *)&sync_trans->updates;
225 }
226
227 keyword_updates = keyword_count == 0 ? NULL :
228 array_front(&sync_trans->keyword_updates);
229 for (i = 0; i < keyword_count; i++) {
230 if (array_is_created(&keyword_updates[i].add_seq)) {
231 synclist = array_append_space(&ctx->sync_list);
232 synclist->array =
233 (const void *)&keyword_updates[i].add_seq;
234 synclist->keyword_idx = i;
235 }
236 if (array_is_created(&keyword_updates[i].remove_seq)) {
237 synclist = array_append_space(&ctx->sync_list);
238 synclist->array =
239 (const void *)&keyword_updates[i].remove_seq;
240 synclist->keyword_idx = i;
241 synclist->keyword_remove = TRUE;
242 }
243 }
244
245 return ret;
246 }
247
248 static bool
mail_index_need_sync(struct mail_index * index,enum mail_index_sync_flags flags,uint32_t log_file_seq,uoff_t log_file_offset)249 mail_index_need_sync(struct mail_index *index, enum mail_index_sync_flags flags,
250 uint32_t log_file_seq, uoff_t log_file_offset)
251 {
252 const struct mail_index_header *hdr = &index->map->hdr;
253 if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0)
254 return TRUE;
255
256 /* sync only if there's something to do */
257 if (hdr->first_recent_uid < hdr->next_uid &&
258 (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0)
259 return TRUE;
260
261 if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 &&
262 (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 &&
263 (index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
264 return TRUE;
265
266 if (log_file_seq == (uint32_t)-1) {
267 /* we want to sync up to transaction log's head */
268 mail_transaction_log_get_head(index->log,
269 &log_file_seq, &log_file_offset);
270 }
271 if ((hdr->log_file_tail_offset < log_file_offset &&
272 hdr->log_file_seq == log_file_seq) ||
273 hdr->log_file_seq < log_file_seq)
274 return TRUE;
275
276 if (index->need_recreate != NULL)
277 return TRUE;
278
279 /* already synced */
280 const char *reason;
281 return mail_cache_need_purge(index->cache, &reason);
282 }
283
284 static int
mail_index_sync_set_log_view(struct mail_index_view * view,uint32_t start_file_seq,uoff_t start_file_offset)285 mail_index_sync_set_log_view(struct mail_index_view *view,
286 uint32_t start_file_seq, uoff_t start_file_offset)
287 {
288 uint32_t log_seq;
289 uoff_t log_offset;
290 const char *reason;
291 bool reset;
292 int ret;
293
294 mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset);
295
296 ret = mail_transaction_log_view_set(view->log_view,
297 start_file_seq, start_file_offset,
298 log_seq, log_offset, &reset, &reason);
299 if (ret < 0)
300 return -1;
301 if (ret == 0) {
302 /* either corrupted or the file was deleted for
303 some reason. either way, we can't go forward */
304 mail_index_set_error(view->index,
305 "Unexpected transaction log desync with index %s: %s",
306 view->index->filepath, reason);
307 return 0;
308 }
309 return 1;
310 }
311
mail_index_sync_begin(struct mail_index * index,struct mail_index_sync_ctx ** ctx_r,struct mail_index_view ** view_r,struct mail_index_transaction ** trans_r,enum mail_index_sync_flags flags)312 int mail_index_sync_begin(struct mail_index *index,
313 struct mail_index_sync_ctx **ctx_r,
314 struct mail_index_view **view_r,
315 struct mail_index_transaction **trans_r,
316 enum mail_index_sync_flags flags)
317 {
318 int ret;
319
320 ret = mail_index_sync_begin_to(index, ctx_r, view_r, trans_r,
321 (uint32_t)-1, UOFF_T_MAX, flags);
322 i_assert(ret != 0 ||
323 (flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) != 0);
324 return ret;
325 }
326
327 static int
mail_index_sync_begin_init(struct mail_index * index,enum mail_index_sync_flags flags,uint32_t log_file_seq,uoff_t log_file_offset)328 mail_index_sync_begin_init(struct mail_index *index,
329 enum mail_index_sync_flags flags,
330 uint32_t log_file_seq, uoff_t log_file_offset)
331 {
332 const struct mail_index_header *hdr;
333 uint32_t seq;
334 uoff_t offset;
335 bool locked = FALSE;
336 int ret;
337
338 /* if we require changes, don't lock transaction log yet. first check
339 if there's anything to sync. */
340 if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0) {
341 if (mail_transaction_log_sync_lock(index->log, "syncing",
342 &seq, &offset) < 0)
343 return -1;
344 locked = TRUE;
345 }
346
347 /* The view must contain what we expect the mailbox to look like
348 currently. That allows the backend to update external flag
349 changes (etc.) if the view doesn't match the mailbox.
350
351 We'll update the view to contain everything that exist in the
352 transaction log except for expunges. They're synced in
353 mail_index_sync_commit(). */
354 if ((ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD)) <= 0) {
355 if (ret == 0) {
356 if (locked)
357 mail_transaction_log_sync_unlock(index->log, "sync init failure");
358 return -1;
359 }
360
361 /* let's try again */
362 if (mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) {
363 if (locked)
364 mail_transaction_log_sync_unlock(index->log, "sync init failure");
365 return -1;
366 }
367 }
368
369 if (!mail_index_need_sync(index, flags, log_file_seq, log_file_offset) &&
370 !index->index_deleted && index->need_recreate == NULL) {
371 if (locked)
372 mail_transaction_log_sync_unlock(index->log, "syncing determined unnecessary");
373 return 0;
374 }
375
376 if (!locked) {
377 /* it looks like we have something to sync. lock the file and
378 check again. */
379 flags &= ENUM_NEGATE(MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES);
380 return mail_index_sync_begin_init(index, flags, log_file_seq,
381 log_file_offset);
382 }
383
384 if (index->index_deleted &&
385 (flags & MAIL_INDEX_SYNC_FLAG_DELETING_INDEX) == 0) {
386 /* index is already deleted. we can't sync. */
387 if (locked)
388 mail_transaction_log_sync_unlock(index->log, "syncing detected deleted index");
389 return -1;
390 }
391
392 hdr = &index->map->hdr;
393 if (hdr->log_file_tail_offset > hdr->log_file_head_offset ||
394 hdr->log_file_seq > seq ||
395 (hdr->log_file_seq == seq && hdr->log_file_tail_offset > offset)) {
396 /* broken sync positions. fix them. */
397 mail_index_set_error(index,
398 "broken sync positions in index file %s",
399 index->filepath);
400 mail_index_fsck_locked(index);
401 }
402 return 1;
403 }
404
405 static int
mail_index_sync_begin_to2(struct mail_index * index,struct mail_index_sync_ctx ** ctx_r,struct mail_index_view ** view_r,struct mail_index_transaction ** trans_r,uint32_t log_file_seq,uoff_t log_file_offset,enum mail_index_sync_flags flags,bool * retry_r)406 mail_index_sync_begin_to2(struct mail_index *index,
407 struct mail_index_sync_ctx **ctx_r,
408 struct mail_index_view **view_r,
409 struct mail_index_transaction **trans_r,
410 uint32_t log_file_seq, uoff_t log_file_offset,
411 enum mail_index_sync_flags flags, bool *retry_r)
412 {
413 const struct mail_index_header *hdr;
414 struct mail_index_sync_ctx *ctx;
415 struct mail_index_view *sync_view;
416 enum mail_index_transaction_flags trans_flags;
417 int ret;
418
419 i_assert(!index->syncing);
420
421 *retry_r = FALSE;
422
423 if (index->map != NULL &&
424 (index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
425 /* index is corrupted and need to be reopened */
426 return -1;
427 }
428
429 if (log_file_seq != (uint32_t)-1)
430 flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
431
432 ret = mail_index_sync_begin_init(index, flags, log_file_seq,
433 log_file_offset);
434 if (ret <= 0)
435 return ret;
436
437 hdr = &index->map->hdr;
438
439 ctx = i_new(struct mail_index_sync_ctx, 1);
440 ctx->index = index;
441 ctx->flags = flags;
442
443 ctx->view = mail_index_view_open(index);
444
445 sync_view = mail_index_dummy_view_open(index);
446 ctx->sync_trans = mail_index_transaction_begin(sync_view,
447 MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
448 mail_index_view_close(&sync_view);
449
450 /* set before any rollbacks are called */
451 index->syncing = TRUE;
452
453 /* we wish to see all the changes from last mailbox sync position to
454 the end of the transaction log */
455 ret = mail_index_sync_set_log_view(ctx->view, hdr->log_file_seq,
456 hdr->log_file_tail_offset);
457 if (ret < 0) {
458 mail_index_sync_rollback(&ctx);
459 return -1;
460 }
461 if (ret == 0) {
462 /* if a log file is missing, there's nothing we can do except
463 to skip over it. fix the problem with fsck and try again. */
464 mail_index_fsck_locked(index);
465 mail_index_sync_rollback(&ctx);
466 *retry_r = TRUE;
467 return 0;
468 }
469
470 /* we need to have all the transactions sorted to optimize
471 caller's mailbox access patterns */
472 if (mail_index_sync_read_and_sort(ctx) < 0) {
473 mail_index_sync_rollback(&ctx);
474 return -1;
475 }
476
477 /* create the transaction after the view has been updated with
478 external transactions and marked as sync view */
479 trans_flags = MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
480 if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES) != 0)
481 trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES;
482 if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_FSYNC) != 0)
483 trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_FSYNC;
484 ctx->ext_trans = mail_index_transaction_begin(ctx->view, trans_flags);
485 ctx->ext_trans->sync_transaction = TRUE;
486 ctx->ext_trans->commit_deleted_index =
487 (flags & (MAIL_INDEX_SYNC_FLAG_DELETING_INDEX |
488 MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX)) != 0;
489
490 *ctx_r = ctx;
491 *view_r = ctx->view;
492 *trans_r = ctx->ext_trans;
493 return 1;
494 }
495
mail_index_sync_begin_to(struct mail_index * index,struct mail_index_sync_ctx ** ctx_r,struct mail_index_view ** view_r,struct mail_index_transaction ** trans_r,uint32_t log_file_seq,uoff_t log_file_offset,enum mail_index_sync_flags flags)496 int mail_index_sync_begin_to(struct mail_index *index,
497 struct mail_index_sync_ctx **ctx_r,
498 struct mail_index_view **view_r,
499 struct mail_index_transaction **trans_r,
500 uint32_t log_file_seq, uoff_t log_file_offset,
501 enum mail_index_sync_flags flags)
502 {
503 bool retry;
504 int ret;
505
506 i_assert(index->open_count > 0);
507
508 ret = mail_index_sync_begin_to2(index, ctx_r, view_r, trans_r,
509 log_file_seq, log_file_offset,
510 flags, &retry);
511 if (retry) {
512 ret = mail_index_sync_begin_to2(index, ctx_r, view_r, trans_r,
513 log_file_seq, log_file_offset,
514 flags, &retry);
515 }
516 return ret;
517 }
518
mail_index_sync_has_expunges(struct mail_index_sync_ctx * ctx)519 bool mail_index_sync_has_expunges(struct mail_index_sync_ctx *ctx)
520 {
521 return array_is_created(&ctx->sync_trans->expunges) &&
522 array_count(&ctx->sync_trans->expunges) > 0;
523 }
524
mail_index_sync_view_have_any(struct mail_index_view * view,enum mail_index_sync_flags flags,bool expunges_only)525 static bool mail_index_sync_view_have_any(struct mail_index_view *view,
526 enum mail_index_sync_flags flags,
527 bool expunges_only)
528 {
529 const struct mail_transaction_header *hdr;
530 const void *data;
531 uint32_t log_seq;
532 uoff_t log_offset;
533 const char *reason;
534 bool reset;
535 int ret;
536
537 if (view->map->hdr.first_recent_uid < view->map->hdr.next_uid &&
538 (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0)
539 return TRUE;
540
541 if ((view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 &&
542 (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 &&
543 (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
544 return TRUE;
545
546 mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset);
547 if (mail_transaction_log_view_set(view->log_view,
548 view->map->hdr.log_file_seq,
549 view->map->hdr.log_file_tail_offset,
550 log_seq, log_offset,
551 &reset, &reason) <= 0) {
552 /* let the actual syncing handle the error */
553 return TRUE;
554 }
555
556 while ((ret = mail_transaction_log_view_next(view->log_view,
557 &hdr, &data)) > 0) {
558 if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
559 continue;
560
561 switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
562 case MAIL_TRANSACTION_EXPUNGE:
563 case MAIL_TRANSACTION_EXPUNGE_GUID:
564 return TRUE;
565 case MAIL_TRANSACTION_EXT_REC_UPDATE:
566 case MAIL_TRANSACTION_EXT_ATOMIC_INC:
567 /* extension record updates aren't exactly needed
568 to be synced, but cache syncing relies on tail
569 offsets being updated. */
570 case MAIL_TRANSACTION_FLAG_UPDATE:
571 case MAIL_TRANSACTION_KEYWORD_UPDATE:
572 case MAIL_TRANSACTION_KEYWORD_RESET:
573 case MAIL_TRANSACTION_INDEX_DELETED:
574 case MAIL_TRANSACTION_INDEX_UNDELETED:
575 if (!expunges_only)
576 return TRUE;
577 break;
578 default:
579 break;
580 }
581 }
582 return ret < 0;
583 }
584
mail_index_sync_have_any(struct mail_index * index,enum mail_index_sync_flags flags)585 bool mail_index_sync_have_any(struct mail_index *index,
586 enum mail_index_sync_flags flags)
587 {
588 struct mail_index_view *view;
589 bool ret;
590
591 view = mail_index_view_open(index);
592 ret = mail_index_sync_view_have_any(view, flags, FALSE);
593 mail_index_view_close(&view);
594 return ret;
595 }
596
mail_index_sync_have_any_expunges(struct mail_index * index)597 bool mail_index_sync_have_any_expunges(struct mail_index *index)
598 {
599 struct mail_index_view *view;
600 bool ret;
601
602 view = mail_index_view_open(index);
603 ret = mail_index_sync_view_have_any(view, 0, TRUE);
604 mail_index_view_close(&view);
605 return ret;
606 }
607
mail_index_sync_get_offsets(struct mail_index_sync_ctx * ctx,uint32_t * seq1_r,uoff_t * offset1_r,uint32_t * seq2_r,uoff_t * offset2_r)608 void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx,
609 uint32_t *seq1_r, uoff_t *offset1_r,
610 uint32_t *seq2_r, uoff_t *offset2_r)
611 {
612 *seq1_r = ctx->view->map->hdr.log_file_seq;
613 *offset1_r = ctx->view->map->hdr.log_file_tail_offset != 0 ?
614 ctx->view->map->hdr.log_file_tail_offset :
615 ctx->view->index->log->head->hdr.hdr_size;
616 mail_transaction_log_get_head(ctx->view->index->log, seq2_r, offset2_r);
617 }
618
619 static void
mail_index_sync_get_expunge(struct mail_index_sync_rec * rec,const struct mail_transaction_expunge_guid * exp)620 mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
621 const struct mail_transaction_expunge_guid *exp)
622 {
623 rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
624 rec->uid1 = exp->uid;
625 rec->uid2 = exp->uid;
626 memcpy(rec->guid_128, exp->guid_128, sizeof(rec->guid_128));
627 }
628
629 static void
mail_index_sync_get_update(struct mail_index_sync_rec * rec,const struct mail_index_flag_update * update)630 mail_index_sync_get_update(struct mail_index_sync_rec *rec,
631 const struct mail_index_flag_update *update)
632 {
633 rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
634 rec->uid1 = update->uid1;
635 rec->uid2 = update->uid2;
636
637 rec->add_flags = update->add_flags;
638 rec->remove_flags = update->remove_flags;
639 }
640
641 static void
mail_index_sync_get_keyword_update(struct mail_index_sync_rec * rec,const struct uid_range * range,struct mail_index_sync_list * sync_list)642 mail_index_sync_get_keyword_update(struct mail_index_sync_rec *rec,
643 const struct uid_range *range,
644 struct mail_index_sync_list *sync_list)
645 {
646 rec->type = !sync_list->keyword_remove ?
647 MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD :
648 MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
649 rec->uid1 = range->uid1;
650 rec->uid2 = range->uid2;
651 rec->keyword_idx = sync_list->keyword_idx;
652 }
653
mail_index_sync_next(struct mail_index_sync_ctx * ctx,struct mail_index_sync_rec * sync_rec)654 bool mail_index_sync_next(struct mail_index_sync_ctx *ctx,
655 struct mail_index_sync_rec *sync_rec)
656 {
657 struct mail_index_transaction *sync_trans = ctx->sync_trans;
658 struct mail_index_sync_list *sync_list;
659 const struct uid_range *uid_range = NULL;
660 unsigned int i, count, next_i;
661 uint32_t next_found_uid;
662
663 next_i = UINT_MAX;
664 next_found_uid = (uint32_t)-1;
665
666 /* FIXME: replace with a priority queue so we don't have to go
667 through the whole list constantly. and remember to make sure that
668 keyword resets are sent before adds! */
669 /* FIXME: pretty ugly to do this for expunges, which isn't even a
670 seq_range. */
671 sync_list = array_get_modifiable(&ctx->sync_list, &count);
672 for (i = 0; i < count; i++) {
673 if (!array_is_created(sync_list[i].array) ||
674 sync_list[i].idx == array_count(sync_list[i].array))
675 continue;
676
677 uid_range = array_idx(sync_list[i].array, sync_list[i].idx);
678 if (uid_range->uid1 == ctx->next_uid) {
679 /* use this one. */
680 break;
681 }
682 if (uid_range->uid1 < next_found_uid) {
683 next_i = i;
684 next_found_uid = uid_range->uid1;
685 }
686 }
687
688 if (i == count) {
689 if (next_i == UINT_MAX) {
690 /* nothing left in sync_list */
691 ctx->fully_synced = TRUE;
692 return FALSE;
693 }
694 ctx->next_uid = next_found_uid;
695 i = next_i;
696 uid_range = array_idx(sync_list[i].array, sync_list[i].idx);
697 }
698
699 if (sync_list[i].array == (void *)&sync_trans->expunges) {
700 mail_index_sync_get_expunge(sync_rec,
701 (const struct mail_transaction_expunge_guid *)uid_range);
702 } else if (sync_list[i].array == (void *)&sync_trans->updates) {
703 mail_index_sync_get_update(sync_rec,
704 (const struct mail_index_flag_update *)uid_range);
705 } else {
706 mail_index_sync_get_keyword_update(sync_rec, uid_range,
707 &sync_list[i]);
708 }
709 sync_list[i].idx++;
710 return TRUE;
711 }
712
mail_index_sync_have_more(struct mail_index_sync_ctx * ctx)713 bool mail_index_sync_have_more(struct mail_index_sync_ctx *ctx)
714 {
715 const struct mail_index_sync_list *sync_list;
716
717 array_foreach(&ctx->sync_list, sync_list) {
718 if (array_is_created(sync_list->array) &&
719 sync_list->idx != array_count(sync_list->array))
720 return TRUE;
721 }
722 return FALSE;
723 }
724
mail_index_sync_set_commit_result(struct mail_index_sync_ctx * ctx,struct mail_index_transaction_commit_result * result)725 void mail_index_sync_set_commit_result(struct mail_index_sync_ctx *ctx,
726 struct mail_index_transaction_commit_result *result)
727 {
728 ctx->sync_commit_result = result;
729 }
730
mail_index_sync_reset(struct mail_index_sync_ctx * ctx)731 void mail_index_sync_reset(struct mail_index_sync_ctx *ctx)
732 {
733 struct mail_index_sync_list *sync_list;
734
735 ctx->next_uid = 0;
736 array_foreach_modifiable(&ctx->sync_list, sync_list)
737 sync_list->idx = 0;
738 }
739
mail_index_sync_no_warning(struct mail_index_sync_ctx * ctx)740 void mail_index_sync_no_warning(struct mail_index_sync_ctx *ctx)
741 {
742 ctx->no_warning = TRUE;
743 }
744
mail_index_sync_set_reason(struct mail_index_sync_ctx * ctx,const char * reason)745 void mail_index_sync_set_reason(struct mail_index_sync_ctx *ctx,
746 const char *reason)
747 {
748 i_free(ctx->reason);
749 ctx->reason = i_strdup(reason);
750 }
751
mail_index_sync_end(struct mail_index_sync_ctx ** _ctx)752 static void mail_index_sync_end(struct mail_index_sync_ctx **_ctx)
753 {
754 struct mail_index_sync_ctx *ctx = *_ctx;
755 const char *lock_reason;
756
757 i_assert(ctx->index->syncing);
758
759 *_ctx = NULL;
760
761 ctx->index->syncing = FALSE;
762 if (ctx->no_warning)
763 lock_reason = NULL;
764 else if (ctx->reason != NULL)
765 lock_reason = ctx->reason;
766 else
767 lock_reason = "Mailbox was synchronized";
768 mail_transaction_log_sync_unlock(ctx->index->log, lock_reason);
769
770 mail_index_view_close(&ctx->view);
771 mail_index_transaction_rollback(&ctx->sync_trans);
772 if (array_is_created(&ctx->sync_list))
773 array_free(&ctx->sync_list);
774 i_free(ctx->reason);
775 i_free(ctx);
776 }
777
778 static void
mail_index_sync_update_mailbox_offset(struct mail_index_sync_ctx * ctx)779 mail_index_sync_update_mailbox_offset(struct mail_index_sync_ctx *ctx)
780 {
781 const struct mail_index_header *hdr = &ctx->index->map->hdr;
782 uint32_t seq;
783 uoff_t offset;
784
785 if (!ctx->fully_synced) {
786 /* Everything wasn't synced. This usually means that syncing
787 was used for locking and nothing was synced. Don't update
788 tail offset. */
789 return;
790 }
791 /* All changes were synced. During the syncing other transactions may
792 have been created and committed as well. They're expected to be
793 external transactions. These could be at least:
794 - mdbox finishing expunges
795 - mdbox writing to dovecot.map.index (requires tail offset updates)
796 - sdbox appending messages
797
798 If any expunges were committed, tail_offset must not be updated
799 before mail_index_map(MAIL_INDEX_SYNC_HANDLER_FILE) is called.
800 Otherwise expunge handlers won't be called for them.
801
802 We'll require MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET flag for the
803 few places that actually require tail_offset to include the
804 externally committed transactions. Otherwise tail_offset is updated
805 only up to what was just synced. */
806 if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET) != 0)
807 mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
808 else {
809 mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
810 &seq, &offset);
811 }
812 mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset);
813
814 /* If tail offset has changed, make sure it gets written to
815 transaction log. do this only if we're required to make changes.
816
817 avoid writing a new tail offset if all the transactions were
818 external, because that wouldn't change effective the tail offset.
819 except e.g. mdbox map requires this to happen, so do it
820 optionally. Also update the tail if we've been calling any expunge
821 handlers, so they won't be called multiple times. That could cause
822 at least cache file's [deleted_]record_count to shrink too much. */
823 if ((hdr->log_file_seq != seq || hdr->log_file_tail_offset < offset) &&
824 (ctx->seen_external_expunges ||
825 ctx->seen_nonexternal_transactions ||
826 (ctx->flags & MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET) != 0)) {
827 ctx->ext_trans->log_updates = TRUE;
828 ctx->ext_trans->tail_offset_changed = TRUE;
829 }
830 }
831
mail_index_sync_want_index_write(struct mail_index * index,const char ** reason_r)832 static bool mail_index_sync_want_index_write(struct mail_index *index, const char **reason_r)
833 {
834 uint32_t log_diff;
835
836 if (index->main_index_hdr_log_file_seq != 0 &&
837 index->main_index_hdr_log_file_seq != index->map->hdr.log_file_seq) {
838 /* dovecot.index points to an old .log file. we were supposed
839 to rewrite the dovecot.index when rotating the log, so
840 we shouldn't usually get here. */
841 *reason_r = "points to old .log file";
842 return TRUE;
843 }
844
845 log_diff = index->map->hdr.log_file_tail_offset -
846 index->main_index_hdr_log_file_tail_offset;
847 if (log_diff > index->optimization_set.index.rewrite_max_log_bytes) {
848 *reason_r = t_strdup_printf(
849 ".log read %u..%u > rewrite_max_log_bytes %"PRIuUOFF_T,
850 index->map->hdr.log_file_tail_offset,
851 index->main_index_hdr_log_file_tail_offset,
852 index->optimization_set.index.rewrite_max_log_bytes);
853 return TRUE;
854 }
855 if (index->index_min_write &&
856 log_diff > index->optimization_set.index.rewrite_min_log_bytes) {
857 *reason_r = t_strdup_printf(
858 ".log read %u..%u > rewrite_min_log_bytes %"PRIuUOFF_T,
859 index->map->hdr.log_file_tail_offset,
860 index->main_index_hdr_log_file_tail_offset,
861 index->optimization_set.index.rewrite_min_log_bytes);
862 return TRUE;
863 }
864
865 if (index->need_recreate != NULL) {
866 *reason_r = t_strdup_printf("Need to recreate index: %s",
867 index->need_recreate);
868 return TRUE;
869 }
870 return FALSE;
871 }
872
mail_index_sync_commit(struct mail_index_sync_ctx ** _ctx)873 int mail_index_sync_commit(struct mail_index_sync_ctx **_ctx)
874 {
875 struct mail_index_sync_ctx *ctx = *_ctx;
876 struct mail_index *index = ctx->index;
877 const char *reason = NULL;
878 uint32_t next_uid;
879 bool want_rotate, index_undeleted, delete_index;
880 int ret = 0, ret2;
881
882 index_undeleted = ctx->ext_trans->index_undeleted;
883 delete_index = index->index_delete_requested && !index_undeleted &&
884 (ctx->flags & (MAIL_INDEX_SYNC_FLAG_DELETING_INDEX |
885 MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX)) != 0;
886 if (delete_index) {
887 /* finish this sync by marking the index deleted */
888 mail_index_set_deleted(ctx->ext_trans);
889 } else if (index->index_deleted && !index_undeleted &&
890 (ctx->flags & MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX) == 0) {
891 /* another process just marked the index deleted.
892 finish the sync, but return error. */
893 mail_index_set_error_nolog(index, "Index is marked deleted");
894 ret = -1;
895 }
896
897 mail_index_sync_update_mailbox_offset(ctx);
898
899 if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0) {
900 next_uid = mail_index_transaction_get_next_uid(ctx->ext_trans);
901 if (index->map->hdr.first_recent_uid < next_uid) {
902 mail_index_update_header(ctx->ext_trans,
903 offsetof(struct mail_index_header,
904 first_recent_uid),
905 &next_uid, sizeof(next_uid), FALSE);
906 }
907 }
908 if (index->hdr_log2_rotate_time_delayed_update != 0) {
909 /* We checked whether .log.2 should be deleted in this same
910 sync. It resulted in wanting to change the log2_rotate_time
911 in the header. Do it here as part of the other changes. */
912 uint32_t log2_rotate_time =
913 index->hdr_log2_rotate_time_delayed_update;
914
915 mail_index_update_header(ctx->ext_trans,
916 offsetof(struct mail_index_header, log2_rotate_time),
917 &log2_rotate_time, sizeof(log2_rotate_time), TRUE);
918 index->hdr_log2_rotate_time_delayed_update = 0;
919 }
920
921 ret2 = mail_index_transaction_commit(&ctx->ext_trans);
922 if (ret2 < 0) {
923 mail_index_sync_end(&ctx);
924 return -1;
925 }
926
927 if (delete_index)
928 index->index_deleted = TRUE;
929 else if (index_undeleted) {
930 index->index_deleted = FALSE;
931 index->index_delete_requested = FALSE;
932 }
933
934 /* refresh the mapping with newly committed external transactions
935 and the synced expunges. sync using file handler here so that the
936 expunge handlers get called. */
937 index->sync_commit_result = ctx->sync_commit_result;
938 if (mail_index_map(ctx->index, MAIL_INDEX_SYNC_HANDLER_FILE) <= 0)
939 ret = -1;
940 index->sync_commit_result = NULL;
941
942 /* The previously called expunged handlers will update cache's
943 record_count and deleted_record_count. That also has a side effect
944 of updating whether cache needs to be purged. */
945 if (ret == 0 && mail_cache_need_purge(index->cache, &reason) &&
946 !mail_cache_transactions_have_changes(index->cache)) {
947 if (mail_cache_purge(index->cache,
948 index->cache->need_purge_file_seq,
949 reason) < 0) {
950 /* can't really do anything if it fails */
951 }
952 /* Make sure the newly committed cache record offsets are
953 updated to the current index. This is important if the
954 dovecot.index gets recreated below, because rotation of
955 dovecot.index.log also re-maps the index to make sure
956 everything is up-to-date. But if it wasn't,
957 mail_index_write() will just assert-crash because
958 log_file_head_offset changed. */
959 if (mail_index_map(ctx->index, MAIL_INDEX_SYNC_HANDLER_FILE) <= 0)
960 ret = -1;
961 }
962
963 /* Log rotation is allowed only if everything was synced. Note that
964 tail_offset might not equal head_offset here, because
965 mail_index_sync_update_mailbox_offset() doesn't always update
966 tail_offset to skip over other committed external transactions.
967 However, it's still safe to do the rotation because external
968 transactions don't require syncing. */
969 want_rotate = ctx->fully_synced &&
970 mail_transaction_log_want_rotate(index->log, &reason);
971 if (ret == 0 &&
972 (want_rotate || mail_index_sync_want_index_write(index, &reason))) {
973 i_free(index->need_recreate);
974 index->index_min_write = FALSE;
975 mail_index_write(index, want_rotate, reason);
976 }
977 mail_index_sync_end(_ctx);
978 return ret;
979 }
980
mail_index_sync_rollback(struct mail_index_sync_ctx ** ctx)981 void mail_index_sync_rollback(struct mail_index_sync_ctx **ctx)
982 {
983 if ((*ctx)->ext_trans != NULL)
984 mail_index_transaction_rollback(&(*ctx)->ext_trans);
985 mail_index_sync_end(ctx);
986 }
987
mail_index_sync_flags_apply(const struct mail_index_sync_rec * sync_rec,uint8_t * flags)988 void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
989 uint8_t *flags)
990 {
991 i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
992
993 *flags = (*flags & ENUM_NEGATE(sync_rec->remove_flags)) | sync_rec->add_flags;
994 }
995
mail_index_sync_keywords_apply(const struct mail_index_sync_rec * sync_rec,ARRAY_TYPE (keyword_indexes)* keywords)996 bool mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec,
997 ARRAY_TYPE(keyword_indexes) *keywords)
998 {
999 const unsigned int *keyword_indexes;
1000 unsigned int idx = sync_rec->keyword_idx;
1001 unsigned int i, count;
1002
1003 keyword_indexes = array_get(keywords, &count);
1004 switch (sync_rec->type) {
1005 case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
1006 for (i = 0; i < count; i++) {
1007 if (keyword_indexes[i] == idx)
1008 return FALSE;
1009 }
1010
1011 array_push_back(keywords, &idx);
1012 return TRUE;
1013 case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
1014 for (i = 0; i < count; i++) {
1015 if (keyword_indexes[i] == idx) {
1016 array_delete(keywords, i, 1);
1017 return TRUE;
1018 }
1019 }
1020 return FALSE;
1021 default:
1022 i_unreached();
1023 return FALSE;
1024 }
1025 }
1026
mail_index_sync_set_corrupted(struct mail_index_sync_map_ctx * ctx,const char * fmt,...)1027 void mail_index_sync_set_corrupted(struct mail_index_sync_map_ctx *ctx,
1028 const char *fmt, ...)
1029 {
1030 va_list va;
1031 uint32_t seq;
1032 uoff_t offset;
1033 char *reason, *reason_free = NULL;
1034
1035 va_start(va, fmt);
1036 reason = reason_free = i_strdup_vprintf(fmt, va);
1037 va_end(va);
1038
1039 ctx->errors = TRUE;
1040 /* make sure we don't get to this same error again by updating the
1041 dovecot.index */
1042 if (ctx->view->index->need_recreate == NULL) {
1043 ctx->view->index->need_recreate = reason;
1044 reason_free = NULL;
1045 }
1046
1047 mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
1048 &seq, &offset);
1049
1050 if (seq < ctx->view->index->fsck_log_head_file_seq ||
1051 (seq == ctx->view->index->fsck_log_head_file_seq &&
1052 offset < ctx->view->index->fsck_log_head_file_offset)) {
1053 /* be silent */
1054 } else {
1055 mail_index_set_error(ctx->view->index,
1056 "Log synchronization error at "
1057 "seq=%u,offset=%"PRIuUOFF_T" for %s: %s",
1058 seq, offset, ctx->view->index->filepath,
1059 reason);
1060 }
1061 i_free(reason_free);
1062 }
1063