1 /* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "array.h"
5 #include "hash.h"
6 #include "istream.h"
7 #include "mail-index-modseq.h"
8 #include "mail-storage-private.h"
9 #include "mail-search-build.h"
10 #include "dsync-transaction-log-scan.h"
11 #include "dsync-mail.h"
12 #include "dsync-mailbox.h"
13 #include "dsync-mailbox-export.h"
14
15 struct dsync_mail_guid_instances {
16 ARRAY_TYPE(seq_range) seqs;
17 bool requested;
18 bool searched;
19 };
20
21 struct dsync_mailbox_exporter {
22 pool_t pool;
23 struct mailbox *box;
24 struct dsync_transaction_log_scan *log_scan;
25 uint32_t last_common_uid;
26
27 struct mailbox_header_lookup_ctx *wanted_headers;
28 struct mailbox_transaction_context *trans;
29 struct mail_search_context *search_ctx;
30 unsigned int search_pos, search_count;
31 unsigned int hdr_hash_version;
32
33 const char *const *hashed_headers;
34
35 /* GUID => instances */
36 HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
37 ARRAY_TYPE(seq_range) requested_uids;
38 ARRAY_TYPE(seq_range) search_uids;
39
40 ARRAY_TYPE(seq_range) expunged_seqs;
41 ARRAY_TYPE(const_string) expunged_guids;
42 unsigned int expunged_guid_idx;
43
44 /* uint32_t UID => struct dsync_mail_change */
45 HASH_TABLE(void *, struct dsync_mail_change *) changes;
46 /* changes sorted by UID */
47 ARRAY(struct dsync_mail_change *) sorted_changes;
48 unsigned int change_idx;
49 uint32_t highest_changed_uid;
50
51 struct mailbox_attribute_iter *attr_iter;
52 struct hash_iterate_context *attr_change_iter;
53 enum mail_attribute_type attr_type;
54 struct dsync_mailbox_attribute attr;
55
56 struct dsync_mail_change change;
57 struct dsync_mail dsync_mail;
58
59 const char *error;
60 enum mail_error mail_error;
61
62 bool body_search_initialized:1;
63 bool auto_export_mails:1;
64 bool mails_have_guids:1;
65 bool minimal_dmail_fill:1;
66 bool return_all_mails:1;
67 bool export_received_timestamps:1;
68 bool export_virtual_sizes:1;
69 bool no_hdr_hashes:1;
70 };
71
dsync_mail_error(struct dsync_mailbox_exporter * exporter,struct mail * mail,const char * field)72 static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
73 struct mail *mail, const char *field)
74 {
75 const char *errstr;
76 enum mail_error error;
77
78 errstr = mailbox_get_last_internal_error(exporter->box, &error);
79 if (error == MAIL_ERROR_EXPUNGED)
80 return 0;
81
82 exporter->mail_error = error;
83 exporter->error = p_strdup_printf(exporter->pool,
84 "Can't lookup %s for UID=%u: %s",
85 field, mail->uid, errstr);
86 return -1;
87 }
88
89 static bool
final_keyword_check(struct dsync_mail_change * change,const char * name,char * type_r)90 final_keyword_check(struct dsync_mail_change *change, const char *name,
91 char *type_r)
92 {
93 const char *const *changes;
94 unsigned int i, count;
95
96 *type_r = KEYWORD_CHANGE_FINAL;
97
98 changes = array_get(&change->keyword_changes, &count);
99 for (i = 0; i < count; i++) {
100 if (strcmp(changes[i]+1, name) != 0)
101 continue;
102
103 switch (changes[i][0]) {
104 case KEYWORD_CHANGE_ADD:
105 /* replace with ADD_AND_FINAL */
106 array_delete(&change->keyword_changes, i, 1);
107 *type_r = KEYWORD_CHANGE_ADD_AND_FINAL;
108 return FALSE;
109 case KEYWORD_CHANGE_REMOVE:
110 /* a final keyword is marked as removed.
111 this shouldn't normally happen. */
112 array_delete(&change->keyword_changes, i, 1);
113 return FALSE;
114 case KEYWORD_CHANGE_ADD_AND_FINAL:
115 case KEYWORD_CHANGE_FINAL:
116 /* no change */
117 return TRUE;
118 }
119 }
120 return FALSE;
121 }
122
123 static void
search_update_flag_changes(struct dsync_mailbox_exporter * exporter,struct mail * mail,struct dsync_mail_change * change)124 search_update_flag_changes(struct dsync_mailbox_exporter *exporter,
125 struct mail *mail, struct dsync_mail_change *change)
126 {
127 const char *const *keywords;
128 unsigned int i;
129 char type;
130
131 i_assert((change->add_flags & change->remove_flags) == 0);
132
133 change->modseq = mail_get_modseq(mail);
134 change->pvt_modseq = mail_get_pvt_modseq(mail);
135 change->final_flags = mail_get_flags(mail);
136
137 keywords = mail_get_keywords(mail);
138 if (!array_is_created(&change->keyword_changes) &&
139 keywords[0] != NULL) {
140 p_array_init(&change->keyword_changes, exporter->pool,
141 str_array_length(keywords));
142 }
143 for (i = 0; keywords[i] != NULL; i++) {
144 /* add the final keyword if it's not already there
145 as +keyword */
146 if (!final_keyword_check(change, keywords[i], &type)) {
147 const char *keyword_change =
148 p_strdup_printf(exporter->pool, "%c%s",
149 type, keywords[i]);
150 array_push_back(&change->keyword_changes,
151 &keyword_change);
152 }
153 }
154 }
155
156 static int
exporter_get_guids(struct dsync_mailbox_exporter * exporter,struct mail * mail,const char ** guid_r,const char ** hdr_hash_r)157 exporter_get_guids(struct dsync_mailbox_exporter *exporter,
158 struct mail *mail, const char **guid_r,
159 const char **hdr_hash_r)
160 {
161 *guid_r = "";
162 *hdr_hash_r = NULL;
163
164 /* always try to get GUID, even if we're also getting header hash */
165 if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0)
166 return dsync_mail_error(exporter, mail, "GUID");
167
168 if (!exporter->mails_have_guids) {
169 /* get header hash also */
170 if (exporter->no_hdr_hashes) {
171 *hdr_hash_r = "";
172 return 1;
173 }
174 if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
175 exporter->hashed_headers, hdr_hash_r) < 0)
176 return dsync_mail_error(exporter, mail, "hdr-stream");
177 return 1;
178 } else if (**guid_r == '\0') {
179 exporter->mail_error = MAIL_ERROR_TEMP;
180 exporter->error = "Backend doesn't support GUIDs, "
181 "sync with header hashes instead";
182 return -1;
183 } else {
184 /* GUIDs are required, we don't need header hash */
185 return 1;
186 }
187 }
188
189 static int
search_update_flag_change_guid(struct dsync_mailbox_exporter * exporter,struct mail * mail)190 search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter,
191 struct mail *mail)
192 {
193 struct dsync_mail_change *change, *log_change;
194 const char *guid, *hdr_hash;
195 int ret;
196
197 change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid));
198 if (change != NULL) {
199 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
200 } else {
201 i_assert(exporter->return_all_mails);
202
203 change = p_new(exporter->pool, struct dsync_mail_change, 1);
204 change->uid = mail->uid;
205 change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
206 hash_table_insert(exporter->changes,
207 POINTER_CAST(mail->uid), change);
208 }
209
210 if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0)
211 return -1;
212 if (ret == 0) {
213 /* the message was expunged during export */
214 i_zero(change);
215 change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
216 change->uid = mail->uid;
217
218 /* find its GUID from log if possible */
219 log_change = dsync_transaction_log_scan_find_new_expunge(
220 exporter->log_scan, mail->uid);
221 if (log_change != NULL)
222 change->guid = log_change->guid;
223 } else {
224 change->guid = *guid == '\0' ? "" :
225 p_strdup(exporter->pool, guid);
226 change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
227 search_update_flag_changes(exporter, mail, change);
228 }
229 return 0;
230 }
231
232 static struct dsync_mail_change *
export_save_change_get(struct dsync_mailbox_exporter * exporter,uint32_t uid)233 export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid)
234 {
235 struct dsync_mail_change *change;
236
237 change = hash_table_lookup(exporter->changes, POINTER_CAST(uid));
238 if (change == NULL) {
239 change = p_new(exporter->pool, struct dsync_mail_change, 1);
240 change->uid = uid;
241 hash_table_insert(exporter->changes, POINTER_CAST(uid), change);
242 } else {
243 /* move flag changes into a save. this happens only when
244 last_common_uid isn't known */
245 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
246 i_assert(exporter->last_common_uid == 0);
247 }
248
249 change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
250 return change;
251 }
252
253 static void
export_add_mail_instance(struct dsync_mailbox_exporter * exporter,struct dsync_mail_change * change,uint32_t seq)254 export_add_mail_instance(struct dsync_mailbox_exporter *exporter,
255 struct dsync_mail_change *change, uint32_t seq)
256 {
257 struct dsync_mail_guid_instances *instances;
258
259 if (exporter->auto_export_mails && !exporter->mails_have_guids) {
260 /* GUIDs not supported, mail is requested by UIDs */
261 seq_range_array_add(&exporter->requested_uids, change->uid);
262 return;
263 }
264 if (*change->guid == '\0') {
265 /* mail UIDs are manually requested */
266 i_assert(!exporter->mails_have_guids);
267 return;
268 }
269
270 instances = hash_table_lookup(exporter->export_guids, change->guid);
271 if (instances == NULL) {
272 instances = p_new(exporter->pool,
273 struct dsync_mail_guid_instances, 1);
274 p_array_init(&instances->seqs, exporter->pool, 2);
275 hash_table_insert(exporter->export_guids,
276 p_strdup(exporter->pool, change->guid),
277 instances);
278 if (exporter->auto_export_mails)
279 instances->requested = TRUE;
280 }
281 seq_range_array_add(&instances->seqs, seq);
282 }
283
284 static int
search_add_save(struct dsync_mailbox_exporter * exporter,struct mail * mail)285 search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
286 {
287 struct dsync_mail_change *change;
288 const char *guid, *hdr_hash;
289 enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
290 time_t received_timestamp = 0;
291 uoff_t virtual_size = UOFF_T_MAX;
292 int ret;
293
294 /* update wanted fields in case we didn't already set them for the
295 search */
296 if (exporter->export_received_timestamps)
297 wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
298 if (exporter->export_virtual_sizes)
299 wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
300 mail_add_temp_wanted_fields(mail, wanted_fields,
301 exporter->wanted_headers);
302
303 /* If message is already expunged here, just skip it */
304 if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
305 return ret;
306
307 if (exporter->export_received_timestamps) {
308 if (mail_get_received_date(mail, &received_timestamp) < 0)
309 return dsync_mail_error(exporter, mail, "received-time");
310 if (received_timestamp == 0) {
311 /* don't allow timestamps to be zero. we want to have
312 asserts verify that the timestamp is set properly. */
313 received_timestamp = 1;
314 }
315 }
316 if (exporter->export_virtual_sizes) {
317 if (mail_get_virtual_size(mail, &virtual_size) < 0)
318 return dsync_mail_error(exporter, mail, "virtual-size");
319 i_assert(virtual_size != UOFF_T_MAX);
320 }
321
322 change = export_save_change_get(exporter, mail->uid);
323 change->guid = *guid == '\0' ? "" :
324 p_strdup(exporter->pool, guid);
325 change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
326 change->received_timestamp = received_timestamp;
327 change->virtual_size = virtual_size;
328 search_update_flag_changes(exporter, mail, change);
329
330 export_add_mail_instance(exporter, change, mail->seq);
331 return 1;
332 }
333
334 static void
dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter * exporter,ARRAY_TYPE (seq_range)* uids)335 dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter,
336 ARRAY_TYPE(seq_range) *uids)
337 {
338 struct hash_iterate_context *iter;
339 void *key;
340 struct dsync_mail_change *change;
341
342 iter = hash_table_iterate_init(exporter->changes);
343 while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
344 if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE)
345 seq_range_array_add(uids, change->uid);
346 }
347 hash_table_iterate_deinit(&iter);
348 }
349
350 static void
dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter * exporter)351 dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter)
352 {
353 struct hash_iterate_context *iter;
354 void *key;
355 struct dsync_mail_change *change;
356
357 /* any flag changes for UIDs above last_common_uid weren't found by
358 mailbox search, which means they were already expunged. for some
359 reason the log scanner found flag changes for the message, but not
360 the expunge. just remove these. */
361 iter = hash_table_iterate_init(exporter->changes);
362 while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
363 if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE &&
364 change->uid > exporter->last_common_uid)
365 hash_table_remove(exporter->changes, key);
366 }
367 hash_table_iterate_deinit(&iter);
368 }
369
370 static void
dsync_mailbox_export_search(struct dsync_mailbox_exporter * exporter)371 dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter)
372 {
373 struct mail_search_context *search_ctx;
374 struct mail_search_args *search_args;
375 struct mail_search_arg *sarg;
376 struct mail *mail;
377 enum mail_fetch_field wanted_fields = 0;
378 struct mailbox_header_lookup_ctx *wanted_headers = NULL;
379 int ret = 0;
380
381 search_args = mail_search_build_init();
382 sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
383 p_array_init(&sarg->value.seqset, search_args->pool, 1);
384
385 if (exporter->return_all_mails || exporter->last_common_uid == 0) {
386 /* we want to know about all mails */
387 seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1);
388 } else {
389 /* lookup GUIDs for messages with flag changes */
390 dsync_mailbox_export_add_flagchange_uids(exporter,
391 &sarg->value.seqset);
392 /* lookup new messages */
393 seq_range_array_add_range(&sarg->value.seqset,
394 exporter->last_common_uid + 1,
395 (uint32_t)-1);
396 }
397
398 if (exporter->last_common_uid == 0) {
399 /* we're syncing all mails, so we can request the wanted
400 fields for all the mails */
401 wanted_fields = MAIL_FETCH_GUID;
402 wanted_headers = exporter->wanted_headers;
403 }
404
405 exporter->trans = mailbox_transaction_begin(exporter->box,
406 MAILBOX_TRANSACTION_FLAG_SYNC,
407 __func__);
408 search_ctx = mailbox_search_init(exporter->trans, search_args, NULL,
409 wanted_fields, wanted_headers);
410 mail_search_args_unref(&search_args);
411
412 while (mailbox_search_next(search_ctx, &mail)) {
413 T_BEGIN {
414 if (mail->uid <= exporter->last_common_uid)
415 ret = search_update_flag_change_guid(exporter, mail);
416 else
417 ret = search_add_save(exporter, mail);
418 } T_END;
419 if (ret < 0)
420 break;
421 }
422 i_assert(ret >= 0 || exporter->error != NULL);
423
424 dsync_mailbox_export_drop_expunged_flag_changes(exporter);
425
426 if (mailbox_search_deinit(&search_ctx) < 0 &&
427 exporter->error == NULL) {
428 exporter->error = p_strdup_printf(exporter->pool,
429 "Mail search failed: %s",
430 mailbox_get_last_internal_error(exporter->box,
431 &exporter->mail_error));
432 }
433 }
434
dsync_mail_change_p_uid_cmp(struct dsync_mail_change * const * c1,struct dsync_mail_change * const * c2)435 static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1,
436 struct dsync_mail_change *const *c2)
437 {
438 if ((*c1)->uid < (*c2)->uid)
439 return -1;
440 if ((*c1)->uid > (*c2)->uid)
441 return 1;
442 return 0;
443 }
444
445 static void
dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter * exporter)446 dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter)
447 {
448 struct hash_iterate_context *iter;
449 void *key;
450 struct dsync_mail_change *change;
451
452 p_array_init(&exporter->sorted_changes, exporter->pool,
453 hash_table_count(exporter->changes));
454
455 iter = hash_table_iterate_init(exporter->changes);
456 while (hash_table_iterate(iter, exporter->changes, &key, &change))
457 array_push_back(&exporter->sorted_changes, &change);
458 hash_table_iterate_deinit(&iter);
459 array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp);
460 }
461
462 static void
dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter * exporter,enum mail_attribute_type type)463 dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
464 enum mail_attribute_type type)
465 {
466 exporter->attr_iter =
467 mailbox_attribute_iter_init(exporter->box, type, "");
468 exporter->attr_type = type;
469 }
470
471 static void
dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter * exporter,struct dsync_transaction_log_scan * log_scan)472 dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter,
473 struct dsync_transaction_log_scan *log_scan)
474 {
475 HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes;
476 struct hash_iterate_context *iter;
477 void *key;
478 struct dsync_mail_change *change, *dup_change;
479
480 log_changes = dsync_transaction_log_scan_get_hash(log_scan);
481 if (dsync_transaction_log_scan_has_all_changes(log_scan)) {
482 /* we tried to access too old/invalid modseqs. to make sure
483 no changes get lost, we need to send all of the messages */
484 exporter->return_all_mails = TRUE;
485 }
486
487 /* clone the hash table, since we're changing it. */
488 hash_table_create_direct(&exporter->changes, exporter->pool,
489 hash_table_count(log_changes));
490 iter = hash_table_iterate_init(log_changes);
491 while (hash_table_iterate(iter, log_changes, &key, &change)) {
492 dup_change = p_new(exporter->pool, struct dsync_mail_change, 1);
493 *dup_change = *change;
494 hash_table_insert(exporter->changes, key, dup_change);
495 if (exporter->highest_changed_uid < change->uid)
496 exporter->highest_changed_uid = change->uid;
497 }
498 hash_table_iterate_deinit(&iter);
499 }
500
501 struct dsync_mailbox_exporter *
dsync_mailbox_export_init(struct mailbox * box,struct dsync_transaction_log_scan * log_scan,uint32_t last_common_uid,enum dsync_mailbox_exporter_flags flags,unsigned int hdr_hash_version,const char * const * hashed_headers)502 dsync_mailbox_export_init(struct mailbox *box,
503 struct dsync_transaction_log_scan *log_scan,
504 uint32_t last_common_uid,
505 enum dsync_mailbox_exporter_flags flags,
506 unsigned int hdr_hash_version,
507 const char *const *hashed_headers)
508 {
509 struct dsync_mailbox_exporter *exporter;
510 pool_t pool;
511
512 pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export",
513 4096);
514 exporter = p_new(pool, struct dsync_mailbox_exporter, 1);
515 exporter->pool = pool;
516 exporter->box = box;
517 exporter->log_scan = log_scan;
518 exporter->last_common_uid = last_common_uid;
519 exporter->auto_export_mails =
520 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
521 exporter->mails_have_guids =
522 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
523 exporter->minimal_dmail_fill =
524 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
525 exporter->export_received_timestamps =
526 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
527 exporter->export_virtual_sizes =
528 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
529 exporter->hdr_hash_version = hdr_hash_version;
530 exporter->no_hdr_hashes =
531 (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
532 exporter->hashed_headers = hashed_headers;
533
534 p_array_init(&exporter->requested_uids, pool, 16);
535 p_array_init(&exporter->search_uids, pool, 16);
536 hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
537 p_array_init(&exporter->expunged_seqs, pool, 16);
538 p_array_init(&exporter->expunged_guids, pool, 16);
539
540 if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
541 exporter->wanted_headers =
542 dsync_mail_get_hash_headers(box, exporter->hashed_headers);
543
544 /* first scan transaction log and save any expunges and flag changes */
545 dsync_mailbox_export_log_scan(exporter, log_scan);
546 /* get saves and also find GUIDs for flag changes */
547 dsync_mailbox_export_search(exporter);
548 /* get the changes sorted by UID */
549 dsync_mailbox_export_sort_changes(exporter);
550
551 dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
552 return exporter;
553 }
554
555 static int
dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter * exporter)556 dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
557 {
558 struct dsync_mailbox_attribute *attr;
559 struct mail_attribute_value value;
560
561 while (hash_table_iterate(exporter->attr_change_iter,
562 dsync_transaction_log_scan_get_attr_hash(exporter->log_scan),
563 &attr, &attr)) {
564 if (attr->exported || !attr->deleted)
565 continue;
566
567 /* lookup the value mainly to get its last_change value. */
568 if (mailbox_attribute_get_stream(exporter->box, attr->type,
569 attr->key, &value) < 0) {
570 exporter->error = p_strdup_printf(exporter->pool,
571 "Mailbox attribute %s lookup failed: %s", attr->key,
572 mailbox_get_last_internal_error(exporter->box,
573 &exporter->mail_error));
574 break;
575 }
576 if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
577 i_stream_unref(&value.value_stream);
578 continue;
579 }
580
581 attr->last_change = value.last_change;
582 if (value.value != NULL || value.value_stream != NULL) {
583 attr->value = p_strdup(exporter->pool, value.value);
584 attr->value_stream = value.value_stream;
585 attr->deleted = FALSE;
586 }
587 attr->exported = TRUE;
588 exporter->attr = *attr;
589 return 1;
590 }
591 hash_table_iterate_deinit(&exporter->attr_change_iter);
592 return 0;
593 }
594
595 static int
dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter * exporter)596 dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
597 {
598 HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
599 struct dsync_mailbox_attribute lookup_attr, *attr;
600 struct dsync_mailbox_attribute *attr_change;
601 const char *key;
602 struct mail_attribute_value value;
603 bool export_all_attrs;
604
605 export_all_attrs = exporter->return_all_mails ||
606 exporter->last_common_uid == 0;
607 attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
608 lookup_attr.type = exporter->attr_type;
609
610 /* note that the order of processing may be important for some
611 attributes. for example sieve can't set a script active until it's
612 first been created */
613 while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
614 lookup_attr.key = key;
615 attr_change = hash_table_lookup(attr_changes, &lookup_attr);
616 if (attr_change == NULL && !export_all_attrs)
617 continue;
618
619 if (mailbox_attribute_get_stream(exporter->box,
620 exporter->attr_type, key,
621 &value) < 0) {
622 exporter->error = p_strdup_printf(exporter->pool,
623 "Mailbox attribute %s lookup failed: %s", key,
624 mailbox_get_last_internal_error(exporter->box,
625 &exporter->mail_error));
626 return -1;
627 }
628 if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
629 /* readonly attributes can't be changed,
630 no point in exporting them */
631 if (value.value_stream != NULL)
632 i_stream_unref(&value.value_stream);
633 continue;
634 }
635 if (value.value == NULL && value.value_stream == NULL &&
636 (attr_change == NULL || !attr_change->deleted)) {
637 /* the attribute was just deleted?
638 skip for this sync. */
639 continue;
640 }
641 if (attr_change != NULL && attr_change->exported) {
642 /* duplicate attribute returned.
643 shouldn't normally happen, but don't crash. */
644 i_warning("Ignoring duplicate attributes '%s'", key);
645 continue;
646 }
647
648 attr = &exporter->attr;
649 i_zero(attr);
650 attr->type = exporter->attr_type;
651 attr->value = p_strdup(exporter->pool, value.value);
652 attr->value_stream = value.value_stream;
653 attr->last_change = value.last_change;
654 if (attr_change != NULL) {
655 attr_change->exported = TRUE;
656 attr->key = attr_change->key;
657 attr->deleted = attr_change->deleted &&
658 !DSYNC_ATTR_HAS_VALUE(attr);
659 attr->modseq = attr_change->modseq;
660 } else {
661 attr->key = p_strdup(exporter->pool, key);
662 }
663 return 1;
664 }
665 if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
666 exporter->error = p_strdup_printf(exporter->pool,
667 "Mailbox attribute iteration failed: %s",
668 mailbox_get_last_internal_error(exporter->box,
669 &exporter->mail_error));
670 return -1;
671 }
672 if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
673 /* export shared attributes */
674 dsync_mailbox_export_attr_init(exporter,
675 MAIL_ATTRIBUTE_TYPE_SHARED);
676 return dsync_mailbox_export_iter_next_attr(exporter);
677 }
678 exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
679 return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
680 }
681
dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter * exporter,const struct dsync_mailbox_attribute ** attr_r)682 int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
683 const struct dsync_mailbox_attribute **attr_r)
684 {
685 int ret;
686
687 if (exporter->error != NULL)
688 return -1;
689
690 i_stream_unref(&exporter->attr.value_stream);
691
692 if (exporter->attr_iter != NULL) {
693 ret = dsync_mailbox_export_iter_next_attr(exporter);
694 } else {
695 ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
696 }
697 if (ret > 0)
698 *attr_r = &exporter->attr;
699 return ret;
700 }
701
dsync_mailbox_export_next(struct dsync_mailbox_exporter * exporter,const struct dsync_mail_change ** change_r)702 int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
703 const struct dsync_mail_change **change_r)
704 {
705 struct dsync_mail_change *const *changes;
706 unsigned int count;
707
708 if (exporter->error != NULL)
709 return -1;
710
711 changes = array_get(&exporter->sorted_changes, &count);
712 if (exporter->change_idx == count)
713 return 0;
714 *change_r = changes[exporter->change_idx++];
715 return 1;
716 }
717
718 static int
dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter * exporter)719 dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
720 {
721 struct mail_search_args *search_args;
722 struct mail_search_arg *sarg;
723 struct hash_iterate_context *iter;
724 const struct seq_range *uids;
725 char *guid;
726 const char *const_guid;
727 enum mail_fetch_field wanted_fields;
728 struct dsync_mail_guid_instances *instances;
729 const struct seq_range *range;
730 unsigned int i, count;
731 uint32_t seq, seq1, seq2;
732
733 i_assert(exporter->search_ctx == NULL);
734
735 search_args = mail_search_build_init();
736 sarg = mail_search_build_add(search_args, SEARCH_SEQSET);
737 p_array_init(&sarg->value.seqset, search_args->pool, 128);
738
739 /* get a list of messages we want to fetch. if there are more than one
740 instance for a GUID, use the first one. */
741 iter = hash_table_iterate_init(exporter->export_guids);
742 while (hash_table_iterate(iter, exporter->export_guids,
743 &guid, &instances)) {
744 if (!instances->requested ||
745 array_count(&instances->seqs) == 0)
746 continue;
747
748 uids = array_front(&instances->seqs);
749 seq = uids[0].seq1;
750 if (!instances->searched) {
751 instances->searched = TRUE;
752 seq_range_array_add(&sarg->value.seqset, seq);
753 } else if (seq_range_exists(&exporter->expunged_seqs, seq)) {
754 /* we're on a second round, refetching expunged
755 messages */
756 seq_range_array_remove(&instances->seqs, seq);
757 seq_range_array_remove(&exporter->expunged_seqs, seq);
758 if (array_count(&instances->seqs) == 0) {
759 /* no instances left */
760 const_guid = guid;
761 array_push_back(&exporter->expunged_guids,
762 &const_guid);
763 continue;
764 }
765 uids = array_front(&instances->seqs);
766 seq = uids[0].seq1;
767 seq_range_array_add(&sarg->value.seqset, seq);
768 }
769 }
770 hash_table_iterate_deinit(&iter);
771
772 /* add requested UIDs */
773 range = array_get(&exporter->requested_uids, &count);
774 for (i = 0; i < count; i++) {
775 mailbox_get_seq_range(exporter->box,
776 range[i].seq1, range[i].seq2,
777 &seq1, &seq2);
778 seq_range_array_add_range(&sarg->value.seqset,
779 seq1, seq2);
780 }
781 array_clear(&exporter->search_uids);
782 array_append_array(&exporter->search_uids, &exporter->requested_uids);
783 array_clear(&exporter->requested_uids);
784
785 wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
786 if (!exporter->minimal_dmail_fill) {
787 wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
788 MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
789 MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
790 }
791 exporter->search_count += seq_range_count(&sarg->value.seqset);
792 exporter->search_ctx =
793 mailbox_search_init(exporter->trans, search_args, NULL,
794 wanted_fields, NULL);
795 mail_search_args_unref(&search_args);
796 return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
797 }
798
799 static void
dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter * exporter)800 dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter)
801 {
802 if (exporter->search_ctx == NULL)
803 return;
804
805 if (mailbox_search_deinit(&exporter->search_ctx) < 0 &&
806 exporter->error == NULL) {
807 exporter->error = p_strdup_printf(exporter->pool,
808 "Mail search failed: %s",
809 mailbox_get_last_internal_error(exporter->box,
810 &exporter->mail_error));
811 }
812 }
813
dsync_mailbox_export_mail(struct dsync_mailbox_exporter * exporter,struct mail * mail)814 static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
815 struct mail *mail)
816 {
817 struct dsync_mail_guid_instances *instances;
818 const char *error_field;
819
820 if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
821 &exporter->dsync_mail, &error_field) < 0)
822 return dsync_mail_error(exporter, mail, error_field);
823
824 instances = *exporter->dsync_mail.guid == '\0' ? NULL :
825 hash_table_lookup(exporter->export_guids,
826 exporter->dsync_mail.guid);
827 if (instances != NULL) {
828 /* GUID found */
829 } else if (exporter->dsync_mail.uid != 0) {
830 /* mail requested by UID */
831 } else {
832 exporter->mail_error = MAIL_ERROR_TEMP;
833 exporter->error = p_strdup_printf(exporter->pool,
834 "GUID unexpectedly changed for UID=%u GUID=%s",
835 mail->uid, exporter->dsync_mail.guid);
836 return -1;
837 }
838
839 if (!seq_range_exists(&exporter->search_uids, mail->uid))
840 exporter->dsync_mail.uid = 0;
841 else
842 exporter->dsync_mail.guid = "";
843
844 /* this message was successfully returned, don't try retrying it */
845 if (instances != NULL)
846 array_clear(&instances->seqs);
847 return 1;
848 }
849
dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter * exporter,const struct dsync_mail_request * request)850 void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
851 const struct dsync_mail_request *request)
852 {
853 struct dsync_mail_guid_instances *instances;
854
855 i_assert(!exporter->auto_export_mails);
856
857 if (request->guid == NULL) {
858 i_assert(request->uid > 0);
859 seq_range_array_add(&exporter->requested_uids, request->uid);
860 return;
861 }
862
863 instances = hash_table_lookup(exporter->export_guids, request->guid);
864 if (instances == NULL) {
865 exporter->mail_error = MAIL_ERROR_TEMP;
866 exporter->error = p_strdup_printf(exporter->pool,
867 "Remote requested unexpected GUID %s", request->guid);
868 return;
869 }
870 instances->requested = TRUE;
871 }
872
dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter * exporter,const struct dsync_mail ** mail_r)873 int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
874 const struct dsync_mail **mail_r)
875 {
876 struct mail *mail;
877 const char *const *guids;
878 unsigned int count;
879 int ret;
880
881 if (exporter->error != NULL)
882 return -1;
883 if (!exporter->body_search_initialized) {
884 exporter->body_search_initialized = TRUE;
885 if (dsync_mailbox_export_body_search_init(exporter) < 0) {
886 i_assert(exporter->error != NULL);
887 return -1;
888 }
889 }
890
891 while (mailbox_search_next(exporter->search_ctx, &mail)) {
892 exporter->search_pos++;
893 if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) {
894 *mail_r = &exporter->dsync_mail;
895 return 1;
896 }
897 if (ret < 0) {
898 i_assert(exporter->error != NULL);
899 return -1;
900 }
901 /* the message was expunged. if the GUID has another instance,
902 try sending it later. */
903 seq_range_array_add(&exporter->expunged_seqs, mail->seq);
904 }
905 /* if some instances of messages were expunged, retry fetching them
906 with other instances */
907 dsync_mailbox_export_body_search_deinit(exporter);
908 if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) {
909 i_assert(exporter->error != NULL);
910 return -1;
911 }
912 if (ret > 0) {
913 /* not finished yet */
914 return dsync_mailbox_export_next_mail(exporter, mail_r);
915 }
916
917 /* finished with messages. if there are any expunged messages,
918 return them */
919 guids = array_get(&exporter->expunged_guids, &count);
920 if (exporter->expunged_guid_idx < count) {
921 i_zero(&exporter->dsync_mail);
922 exporter->dsync_mail.guid =
923 guids[exporter->expunged_guid_idx++];
924 *mail_r = &exporter->dsync_mail;
925 return 1;
926 }
927 return 0;
928 }
929
dsync_mailbox_export_deinit(struct dsync_mailbox_exporter ** _exporter,const char ** errstr_r,enum mail_error * error_r)930 int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter,
931 const char **errstr_r, enum mail_error *error_r)
932 {
933 struct dsync_mailbox_exporter *exporter = *_exporter;
934
935 *_exporter = NULL;
936
937 if (exporter->attr_iter != NULL)
938 (void)mailbox_attribute_iter_deinit(&exporter->attr_iter);
939 dsync_mailbox_export_body_search_deinit(exporter);
940 (void)mailbox_transaction_commit(&exporter->trans);
941 mailbox_header_lookup_unref(&exporter->wanted_headers);
942
943 i_stream_unref(&exporter->attr.value_stream);
944 hash_table_destroy(&exporter->export_guids);
945 hash_table_destroy(&exporter->changes);
946
947 i_assert((exporter->error != NULL) == (exporter->mail_error != 0));
948
949 *error_r = exporter->mail_error;
950 *errstr_r = t_strdup(exporter->error);
951 pool_unref(&exporter->pool);
952 return *errstr_r != NULL ? -1 : 0;
953 }
954
dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter * exporter)955 const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter)
956 {
957 if (exporter->search_ctx == NULL)
958 return "";
959 return t_strdup_printf("%u/%u", exporter->search_pos,
960 exporter->search_count);
961 }
962