1 /* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
2 
3 #include "lib.h"
4 #include "array.h"
5 #include "mail-transaction-log-private.h"
6 #include "mail-index-private.h"
7 #include "mail-index-sync-private.h"
8 #include "mail-index-modseq.h"
9 
10 ARRAY_DEFINE_TYPE(modseqs, uint64_t);
11 
12 enum modseq_metadata_idx {
13 	/* must be in the same order as enum mail_flags */
14 	METADATA_MODSEQ_IDX_ANSWERED = 0,
15 	METADATA_MODSEQ_IDX_FLAGGED,
16 	METADATA_MODSEQ_IDX_DELETED,
17 	METADATA_MODSEQ_IDX_SEEN,
18 	METADATA_MODSEQ_IDX_DRAFT,
19 
20 	METADATA_MODSEQ_IDX_KEYWORD_START
21 };
22 
23 struct metadata_modseqs {
24 	ARRAY_TYPE(modseqs) modseqs;
25 };
26 
27 struct mail_index_map_modseq {
28 	/* indexes use enum modseq_metadata_idx */
29 	ARRAY(struct metadata_modseqs) metadata_modseqs;
30 };
31 
32 struct mail_index_modseq_sync {
33 	struct mail_index_sync_map_ctx *sync_map_ctx;
34 	struct mail_index_view *view;
35 	struct mail_transaction_log_view *log_view;
36 	struct mail_index_map_modseq *mmap;
37 };
38 
mail_index_modseq_init(struct mail_index * index)39 void mail_index_modseq_init(struct mail_index *index)
40 {
41 	index->modseq_ext_id =
42 		mail_index_ext_register(index, MAIL_INDEX_MODSEQ_EXT_NAME,
43 					sizeof(struct mail_index_modseq_header),
44 					sizeof(uint64_t), sizeof(uint64_t));
45 }
46 
mail_index_modseq_get_head(struct mail_index * index)47 static uint64_t mail_index_modseq_get_head(struct mail_index *index)
48 {
49 	return index->log->head == NULL ? 1 :
50 		I_MAX(index->log->head->sync_highest_modseq, 1);
51 }
52 
mail_index_modseq_enable(struct mail_index * index)53 void mail_index_modseq_enable(struct mail_index *index)
54 {
55 	struct mail_index_transaction *trans;
56 	struct mail_index_view *view;
57 	struct mail_index_modseq_header hdr;
58 	uint32_t ext_map_idx;
59 
60 	if (index->modseqs_enabled)
61 		return;
62 
63 	if (!mail_index_map_get_ext_idx(index->map, index->modseq_ext_id,
64 					&ext_map_idx)) {
65 		/* modseqs not enabled to the index yet, add them. */
66 		view = mail_index_view_open(index);
67 		trans = mail_index_transaction_begin(view, 0);
68 
69 		i_zero(&hdr);
70 		hdr.highest_modseq = mail_index_modseq_get_head(index);
71 		mail_index_update_header_ext(trans, index->modseq_ext_id,
72 					     0, &hdr, sizeof(hdr));
73 
74 		/* commit also refreshes the index, which syncs the modseqs */
75 		(void)mail_index_transaction_commit(&trans);
76 		mail_index_view_close(&view);
77 
78 		/* get the modseq extension to index map */
79 		if (!mail_index_map_get_ext_idx(index->map,
80 						index->modseq_ext_id,
81 						&ext_map_idx)) {
82 			/* didn't work for some reason */
83 			return;
84 		}
85 	}
86 	index->modseqs_enabled = TRUE;
87 }
88 
mail_index_have_modseq_tracking(struct mail_index * index)89 bool mail_index_have_modseq_tracking(struct mail_index *index)
90 {
91 	return mail_index_map_get_modseq_header(index->map) != NULL;
92 }
93 
94 const struct mail_index_modseq_header *
mail_index_map_get_modseq_header(struct mail_index_map * map)95 mail_index_map_get_modseq_header(struct mail_index_map *map)
96 {
97 	const struct mail_index_ext *ext;
98 	uint32_t idx;
99 
100 	if (!mail_index_map_get_ext_idx(map, map->index->modseq_ext_id, &idx))
101 		return NULL;
102 
103 	ext = array_idx(&map->extensions, idx);
104 	if (ext->hdr_size != sizeof(struct mail_index_modseq_header))
105 		return NULL;
106 
107 	return MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
108 }
109 
mail_index_map_modseq_get_highest(struct mail_index_map * map)110 uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map)
111 {
112 	const struct mail_index_modseq_header *modseq_hdr;
113 
114 	modseq_hdr = mail_index_map_get_modseq_header(map);
115 	if (modseq_hdr != NULL && modseq_hdr->highest_modseq != 0)
116 		return modseq_hdr->highest_modseq;
117 	else {
118 		/* fallback to returning the log head. if modseqs aren't
119 		   enabled, we return 0. */
120 		return map->index->log->head == NULL ? 0 :
121 			map->index->log->head->sync_highest_modseq;
122 	}
123 }
124 
mail_index_modseq_get_highest(struct mail_index_view * view)125 uint64_t mail_index_modseq_get_highest(struct mail_index_view *view)
126 {
127 	return mail_index_map_modseq_get_highest(view->map);
128 }
129 
130 static struct mail_index_map_modseq *
mail_index_map_modseq(struct mail_index_view * view)131 mail_index_map_modseq(struct mail_index_view *view)
132 {
133 	struct mail_index_map_modseq *mmap = view->map->rec_map->modseq;
134 	uint32_t ext_map_idx;
135 
136 	if (mmap != NULL)
137 		return mmap;
138 
139 	/* don't start tracking until we've seen modseq extension intro */
140 	if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
141 					&ext_map_idx))
142 		return NULL;
143 
144 	mmap = i_new(struct mail_index_map_modseq, 1);
145 	i_array_init(&mmap->metadata_modseqs,
146 		     METADATA_MODSEQ_IDX_KEYWORD_START +
147 		     array_count(&view->index->keywords));
148 	view->map->rec_map->modseq = mmap;
149 	return mmap;
150 }
151 
mail_index_modseq_lookup(struct mail_index_view * view,uint32_t seq)152 uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq)
153 {
154 	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
155 	struct mail_index_map *map;
156 	const struct mail_index_ext *ext;
157 	const struct mail_index_record *rec;
158 	const uint64_t *modseqp;
159 	uint32_t ext_map_idx;
160 
161 	if (mmap == NULL)
162 		return mail_index_modseq_get_head(view->index);
163 
164 	rec = mail_index_lookup_full(view, seq, &map);
165 	if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
166 					&ext_map_idx)) {
167 		/* not enabled yet */
168 		return mail_index_modseq_get_head(view->index);
169 	}
170 
171 	ext = array_idx(&map->extensions, ext_map_idx);
172 	modseqp = CONST_PTR_OFFSET(rec, ext->record_offset);
173 	if (*modseqp == 0) {
174 		/* If we're here because we just enabled modseqs, we'll return
175 		   the same modseq (initial highestmodseq) for all messages.
176 		   The next sync will change these zeros to initial
177 		   highestmodseq or higher.
178 
179 		   If we're here because a message got appended but modseq
180 		   wasn't set (older Dovecot?), we'll again use the current
181 		   highest modseq. This isn't exactly correct, but it gets
182 		   fixed after the next sync and this situation shouldn't
183 		   normally happen anyway. */
184 		return mail_index_modseq_get_highest(view);
185 	}
186 	return *modseqp;
187 }
188 
mail_index_modseq_set(struct mail_index_view * view,uint32_t seq,uint64_t min_modseq)189 int mail_index_modseq_set(struct mail_index_view *view,
190 			  uint32_t seq, uint64_t min_modseq)
191 {
192 	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
193 	const struct mail_index_ext *ext;
194 	struct mail_index_record *rec;
195 	uint64_t *modseqp;
196 	uint32_t ext_map_idx;
197 
198 	if (mmap == NULL)
199 		return -1;
200 
201 	rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
202 	if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
203 					&ext_map_idx))
204 		return -1;
205 
206 	ext = array_idx(&view->map->extensions, ext_map_idx);
207 	modseqp = PTR_OFFSET(rec, ext->record_offset);
208 	if (*modseqp > min_modseq)
209 		return 0;
210 	else {
211 		*modseqp = min_modseq;
212 		return 1;
213 	}
214 }
215 
216 static uint64_t
modseq_idx_lookup(struct mail_index_map_modseq * mmap,unsigned int idx,uint32_t seq)217 modseq_idx_lookup(struct mail_index_map_modseq *mmap,
218 		  unsigned int idx, uint32_t seq)
219 {
220 	const struct metadata_modseqs *metadata;
221 	const uint64_t *modseqs;
222 	unsigned int count;
223 
224 	metadata = array_get(&mmap->metadata_modseqs, &count);
225 	if (idx >= count || !array_is_created(&metadata[idx].modseqs))
226 		return 0;
227 
228 	modseqs = array_get(&metadata[idx].modseqs, &count);
229 	return seq > count ? 0 : modseqs[seq-1];
230 }
231 
mail_index_modseq_lookup_flags(struct mail_index_view * view,enum mail_flags flags_mask,uint32_t seq)232 uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view,
233 					enum mail_flags flags_mask,
234 					uint32_t seq)
235 {
236 	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
237 	unsigned int i;
238 	uint64_t modseq, highest_modseq = 0;
239 
240 	if (mmap != NULL) {
241 		/* first try to find a specific match */
242 		for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
243 			if ((flags_mask & (1 << i)) != 0) {
244 				modseq = modseq_idx_lookup(mmap, i, seq);
245 				if (highest_modseq < modseq)
246 					highest_modseq = modseq;
247 			}
248 		}
249 	}
250 
251 	if (highest_modseq == 0) {
252 		/* no specific matches, fallback to using the highest */
253 		highest_modseq = mail_index_modseq_lookup(view, seq);
254 	}
255 	return highest_modseq;
256 }
257 
mail_index_modseq_lookup_keywords(struct mail_index_view * view,const struct mail_keywords * keywords,uint32_t seq)258 uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view,
259 					   const struct mail_keywords *keywords,
260 					   uint32_t seq)
261 {
262 	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
263 	unsigned int i, metadata_idx;
264 	uint64_t modseq, highest_modseq = 0;
265 
266 	if (mmap != NULL) {
267 		/* first try to find a specific match */
268 		for (i = 0; i < keywords->count; i++) {
269 			metadata_idx = METADATA_MODSEQ_IDX_KEYWORD_START +
270 				keywords->idx[i];
271 
272 			modseq = modseq_idx_lookup(mmap, metadata_idx, seq);
273 			if (highest_modseq < modseq)
274 				highest_modseq = modseq;
275 		}
276 	}
277 
278 	if (highest_modseq == 0) {
279 		/* no specific matches, fallback to using the highest */
280 		highest_modseq = mail_index_modseq_lookup(view, seq);
281 	}
282 	return highest_modseq;
283 }
284 
285 static void
mail_index_modseq_update(struct mail_index_modseq_sync * ctx,uint64_t modseq,bool nonzeros,uint32_t seq1,uint32_t seq2)286 mail_index_modseq_update(struct mail_index_modseq_sync *ctx,
287 			 uint64_t modseq, bool nonzeros,
288 			 uint32_t seq1, uint32_t seq2)
289 {
290 	const struct mail_index_ext *ext;
291 	struct mail_index_record *rec;
292 	uint32_t ext_map_idx;
293 	uint64_t *modseqp;
294 
295 	if (!mail_index_map_get_ext_idx(ctx->view->map,
296 					ctx->view->index->modseq_ext_id,
297 					&ext_map_idx))
298 		return;
299 
300 	ext = array_idx(&ctx->view->map->extensions, ext_map_idx);
301 	for (; seq1 <= seq2; seq1++) {
302 		rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq1);
303 		modseqp = PTR_OFFSET(rec, ext->record_offset);
304 		if (*modseqp == 0 || (nonzeros && *modseqp < modseq))
305 			*modseqp = modseq;
306 	}
307 }
308 
309 static bool
mail_index_modseq_update_to_highest(struct mail_index_modseq_sync * ctx,uint32_t seq1,uint32_t seq2)310 mail_index_modseq_update_to_highest(struct mail_index_modseq_sync *ctx,
311 				    uint32_t seq1, uint32_t seq2)
312 {
313 	uint64_t modseq;
314 
315 	if (ctx->mmap == NULL)
316 		return FALSE;
317 
318 	modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
319 	mail_index_modseq_update(ctx, modseq, TRUE, seq1, seq2);
320 	return TRUE;
321 }
322 
323 static void
mail_index_modseq_update_old_rec(struct mail_index_modseq_sync * ctx,const struct mail_transaction_header * thdr,const void * tdata)324 mail_index_modseq_update_old_rec(struct mail_index_modseq_sync *ctx,
325 				 const struct mail_transaction_header *thdr,
326 				 const void *tdata)
327 {
328 	ARRAY_TYPE(seq_range) uids = ARRAY_INIT;
329 	const struct seq_range *rec;
330 	buffer_t uid_buf;
331 	unsigned int i, count;
332 	uint32_t seq1, seq2;
333 
334 	switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) {
335 	case MAIL_TRANSACTION_APPEND: {
336 		const struct mail_index_record *appends = tdata;
337 
338 		count = thdr->size / sizeof(*appends);
339 		for (i = 0; i < count; i++) {
340 			if (mail_index_lookup_seq(ctx->view,
341 						  appends[i].uid, &seq1)) {
342 				(void)mail_index_modseq_update_to_highest(ctx, seq1, seq1);
343 			}
344 		}
345 		return;
346 	}
347 	case MAIL_TRANSACTION_FLAG_UPDATE: {
348 		buffer_create_from_const_data(&uid_buf, tdata, thdr->size);
349 		array_create_from_buffer(&uids, &uid_buf,
350 			sizeof(struct mail_transaction_flag_update));
351 		break;
352 	}
353 	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
354 		const struct mail_transaction_keyword_update *rec = tdata;
355 		unsigned int seqset_offset;
356 
357 		seqset_offset = sizeof(*rec) + rec->name_size;
358 		if ((seqset_offset % 4) != 0)
359 			seqset_offset += 4 - (seqset_offset % 4);
360 
361 		buffer_create_from_const_data(&uid_buf,
362 					 CONST_PTR_OFFSET(tdata, seqset_offset),
363 					 thdr->size - seqset_offset);
364 		array_create_from_buffer(&uids, &uid_buf, sizeof(uint32_t)*2);
365 		break;
366 	}
367 	case MAIL_TRANSACTION_KEYWORD_RESET:
368 		buffer_create_from_const_data(&uid_buf, tdata, thdr->size);
369 		array_create_from_buffer(&uids, &uid_buf,
370 			sizeof(struct mail_transaction_keyword_reset));
371 		break;
372 	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
373 		break;
374 	default:
375 		return;
376 	}
377 
378 	/* update modseqs */
379 	count = array_is_created(&uids) ? array_count(&uids) : 0;
380 	for (i = 0; i < count; i++) {
381 		rec = array_idx(&uids, i);
382 		if (mail_index_lookup_seq_range(ctx->view, rec->seq1, rec->seq2,
383 						&seq1, &seq2))
384 			(void)mail_index_modseq_update_to_highest(ctx, seq1, seq2);
385 	}
386 }
387 
mail_index_modseq_sync_init(struct mail_index_modseq_sync * ctx)388 static void mail_index_modseq_sync_init(struct mail_index_modseq_sync *ctx)
389 {
390 	struct mail_index_map *map = ctx->view->map;
391 	const struct mail_index_ext *ext;
392 	const struct mail_index_modseq_header *hdr;
393 	const struct mail_transaction_header *thdr;
394 	const void *tdata;
395 	const char *reason;
396 	uint32_t ext_map_idx;
397 	uint32_t end_seq;
398 	uoff_t end_offset;
399 	uint64_t cur_modseq;
400 	bool reset;
401 	int ret;
402 
403 	if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id,
404 					&ext_map_idx))
405 		i_unreached();
406 	ext = array_idx(&map->extensions, ext_map_idx);
407 
408 	/* get the current highest_modseq. don't change any modseq below it. */
409 	hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
410 
411 	/* Scan logs for updates between ext_hdr.log_* .. view position.
412 	   There are two reasons why there could be any:
413 
414 	   1) We just enabled modseqs and we're filling the initial values.
415 	   2) A non-modseq-aware Dovecot version added new messages and wrote
416 	      dovecot.index file. */
417 	mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
418 					       &end_seq, &end_offset);
419 	if (end_seq < hdr->log_seq ||
420 	    (end_seq == hdr->log_seq && end_offset <= hdr->log_offset)) {
421 		/* modseqs are up to date */
422 		return;
423 	}
424 
425 	ctx->log_view = mail_transaction_log_view_open(ctx->view->index->log);
426 	ret = mail_transaction_log_view_set(ctx->log_view,
427 					    I_MAX(1, hdr->log_seq),
428 					    hdr->log_offset,
429 					    end_seq, end_offset, &reset, &reason);
430 	if (ret <= 0) {
431 		/* missing files / error - try with only the last file */
432 		ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0,
433 						    end_seq, end_offset,
434 						    &reset, &reason);
435 		/* since we don't know if we skipped some changes, set all
436 		   modseqs to beginning of the latest file. */
437 		cur_modseq = mail_transaction_log_view_get_prev_modseq(
438 								ctx->log_view);
439 		if (cur_modseq < hdr->highest_modseq) {
440 			/* should happen only when setting initial modseqs.
441 			   we may already have returned highest_modseq as
442 			   some messages' modseq value. don't shrink it. */
443 			cur_modseq = hdr->highest_modseq;
444 		}
445 		mail_index_modseq_update(ctx, cur_modseq, TRUE, 1,
446 					 map->hdr.messages_count);
447 	} else {
448 		/* we have all the logs. replace zero modseqs with the current
449 		   highest modseq (we may have already returned it for them). */
450 		mail_index_modseq_update(ctx, hdr->highest_modseq, FALSE, 1,
451 					 map->hdr.messages_count);
452 	}
453 	if (ret > 0) {
454 		while (mail_transaction_log_view_next(ctx->log_view,
455 						      &thdr, &tdata) > 0) {
456 			T_BEGIN {
457 				mail_index_modseq_update_old_rec(ctx, thdr,
458 								 tdata);
459 			} T_END;
460 		}
461 	}
462 	mail_transaction_log_view_close(&ctx->log_view);
463 }
464 
465 struct mail_index_modseq_sync *
mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx * sync_map_ctx)466 mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx)
467 {
468 	struct mail_index_modseq_sync *ctx;
469 
470 	ctx = i_new(struct mail_index_modseq_sync, 1);
471 	ctx->sync_map_ctx = sync_map_ctx;
472 	ctx->view = sync_map_ctx->view;
473 	ctx->mmap = mail_index_map_modseq(ctx->view);
474 	if (ctx->mmap != NULL) {
475 		mail_index_modseq_sync_init(ctx);
476 		ctx->log_view = ctx->view->log_view;
477 	}
478 	return ctx;
479 }
480 
mail_index_modseq_update_header(struct mail_index_modseq_sync * ctx)481 static void mail_index_modseq_update_header(struct mail_index_modseq_sync *ctx)
482 {
483 	struct mail_index_view *view = ctx->view;
484 	struct mail_index_map *map = view->map;
485 	const struct mail_index_ext *ext;
486 	const struct mail_index_modseq_header *old_modseq_hdr;
487 	struct mail_index_modseq_header new_modseq_hdr;
488 	uint32_t ext_map_idx, log_seq;
489 	uoff_t log_offset;
490 	uint64_t highest_modseq;
491 
492 	if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
493 					&ext_map_idx))
494 		return;
495 
496 	mail_transaction_log_view_get_prev_pos(view->log_view,
497 					       &log_seq, &log_offset);
498 	highest_modseq = mail_transaction_log_view_get_prev_modseq(view->log_view);
499 
500 	ext = array_idx(&map->extensions, ext_map_idx);
501 	old_modseq_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
502 
503 	if (old_modseq_hdr->log_seq < log_seq ||
504 	    (old_modseq_hdr->log_seq == log_seq &&
505 	     old_modseq_hdr->log_offset < log_offset)) {
506 		i_zero(&new_modseq_hdr);
507 		new_modseq_hdr.highest_modseq = highest_modseq;
508 		new_modseq_hdr.log_seq = log_seq;
509 		new_modseq_hdr.log_offset = log_offset;
510 
511 		buffer_write(map->hdr_copy_buf, ext->hdr_offset,
512 			     &new_modseq_hdr, sizeof(new_modseq_hdr));
513 		i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
514 	}
515 }
516 
mail_index_modseq_sync_end(struct mail_index_modseq_sync ** _ctx)517 void mail_index_modseq_sync_end(struct mail_index_modseq_sync **_ctx)
518 {
519 	struct mail_index_modseq_sync *ctx = *_ctx;
520 
521 	*_ctx = NULL;
522 	if (ctx->mmap != NULL) {
523 		i_assert(ctx->mmap == ctx->view->map->rec_map->modseq);
524 		mail_index_modseq_update_header(ctx);
525 	}
526 	i_free(ctx);
527 }
528 
mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync * ctx)529 void mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync *ctx)
530 {
531 	ctx->mmap = mail_index_map_modseq(ctx->view);
532 }
533 
mail_index_modseq_hdr_update(struct mail_index_modseq_sync * ctx)534 void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx)
535 {
536 	if (ctx->mmap == NULL) {
537 		ctx->mmap = mail_index_map_modseq(ctx->view);
538 		i_assert(ctx->mmap != NULL);
539 		mail_index_modseq_sync_init(ctx);
540 		ctx->log_view = ctx->view->log_view;
541 	}
542 }
543 
mail_index_modseq_append(struct mail_index_modseq_sync * ctx,uint32_t seq)544 void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq)
545 {
546 	(void)mail_index_modseq_update_to_highest(ctx, seq, seq);
547 }
548 
mail_index_modseq_expunge(struct mail_index_modseq_sync * ctx,uint32_t seq1,uint32_t seq2)549 void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx,
550 			       uint32_t seq1, uint32_t seq2)
551 {
552 	struct metadata_modseqs *metadata;
553 
554 	if (ctx->mmap == NULL)
555 		return;
556 
557 	seq1--;
558 	array_foreach_modifiable(&ctx->mmap->metadata_modseqs, metadata) {
559 		if (array_is_created(&metadata->modseqs))
560 			array_delete(&metadata->modseqs, seq1, seq2-seq1);
561 	}
562 }
563 
564 static void
modseqs_update(ARRAY_TYPE (modseqs)* array,uint32_t seq1,uint32_t seq2,uint64_t value)565 modseqs_update(ARRAY_TYPE(modseqs) *array, uint32_t seq1, uint32_t seq2,
566 	       uint64_t value)
567 {
568 	uint64_t *modseqp;
569 
570 	for (; seq1 <= seq2; seq1++) {
571 		modseqp = array_idx_get_space(array, seq1-1);
572 		if (*modseqp < value)
573 			*modseqp = value;
574 	}
575 }
576 
577 static void
modseqs_idx_update(struct mail_index_modseq_sync * ctx,unsigned int idx,uint32_t seq1,uint32_t seq2)578 modseqs_idx_update(struct mail_index_modseq_sync *ctx, unsigned int idx,
579 		   uint32_t seq1, uint32_t seq2)
580 {
581 	struct metadata_modseqs *metadata;
582 	uint64_t modseq;
583 
584 	if (!ctx->view->index->modseqs_enabled) {
585 		/* we want to keep permanent modseqs updated, but don't bother
586 		   updating in-memory per-flag updates */
587 		return;
588 	}
589 
590 	modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
591 	metadata = array_idx_get_space(&ctx->mmap->metadata_modseqs, idx);
592 	if (!array_is_created(&metadata->modseqs))
593 		i_array_init(&metadata->modseqs, seq2 + 16);
594 	modseqs_update(&metadata->modseqs, seq1, seq2, modseq);
595 }
596 
mail_index_modseq_update_flags(struct mail_index_modseq_sync * ctx,enum mail_flags flags_mask,uint32_t seq1,uint32_t seq2)597 void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx,
598 				    enum mail_flags flags_mask,
599 				    uint32_t seq1, uint32_t seq2)
600 {
601 	unsigned int i;
602 
603 	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
604 		return;
605 
606 	for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
607 		if ((flags_mask & (1 << i)) != 0)
608 			modseqs_idx_update(ctx, i, seq1, seq2);
609 	}
610 }
611 
mail_index_modseq_update_keyword(struct mail_index_modseq_sync * ctx,unsigned int keyword_idx,uint32_t seq1,uint32_t seq2)612 void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx,
613 				      unsigned int keyword_idx,
614 				      uint32_t seq1, uint32_t seq2)
615 {
616 	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
617 		return;
618 
619 	modseqs_idx_update(ctx, METADATA_MODSEQ_IDX_KEYWORD_START + keyword_idx,
620 			   seq1, seq2);
621 }
622 
mail_index_modseq_reset_keywords(struct mail_index_modseq_sync * ctx,uint32_t seq1,uint32_t seq2)623 void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx,
624 				      uint32_t seq1, uint32_t seq2)
625 {
626 	unsigned int i, count;
627 
628 	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
629 		return;
630 
631 	count = array_count(&ctx->mmap->metadata_modseqs);
632 	for (i = METADATA_MODSEQ_IDX_KEYWORD_START; i < count; i++)
633 		modseqs_idx_update(ctx, i, seq1, seq2);
634 }
635 
636 struct mail_index_map_modseq *
mail_index_map_modseq_clone(const struct mail_index_map_modseq * mmap)637 mail_index_map_modseq_clone(const struct mail_index_map_modseq *mmap)
638 {
639 	struct mail_index_map_modseq *new_mmap;
640 	const struct metadata_modseqs *src_metadata;
641 	struct metadata_modseqs *dest_metadata;
642 	unsigned int i, count;
643 
644 	src_metadata = array_get(&mmap->metadata_modseqs, &count);
645 
646 	new_mmap = i_new(struct mail_index_map_modseq, 1);
647 	i_array_init(&new_mmap->metadata_modseqs, count + 16);
648 
649 	for (i = 0; i < count; i++) {
650 		dest_metadata = array_append_space(&new_mmap->metadata_modseqs);
651 		if (array_is_created(&src_metadata[i].modseqs)) {
652 			i_array_init(&dest_metadata->modseqs,
653 				     array_count(&src_metadata[i].modseqs));
654 			array_append_array(&dest_metadata->modseqs,
655 					   &src_metadata[i].modseqs);
656 		}
657 	}
658 	return new_mmap;
659 }
660 
mail_index_map_modseq_free(struct mail_index_map_modseq ** _mmap)661 void mail_index_map_modseq_free(struct mail_index_map_modseq **_mmap)
662 {
663 	struct mail_index_map_modseq *mmap = *_mmap;
664 	struct metadata_modseqs *metadata;
665 
666 	*_mmap = NULL;
667 
668 	array_foreach_modifiable(&mmap->metadata_modseqs, metadata) {
669 		if (array_is_created(&metadata->modseqs))
670 			array_free(&metadata->modseqs);
671 	}
672 	array_free(&mmap->metadata_modseqs);
673 	i_free(mmap);
674 }
675 
mail_index_modseq_get_next_log_offset(struct mail_index_view * view,uint64_t modseq,uint32_t * log_seq_r,uoff_t * log_offset_r)676 bool mail_index_modseq_get_next_log_offset(struct mail_index_view *view,
677 					   uint64_t modseq, uint32_t *log_seq_r,
678 					   uoff_t *log_offset_r)
679 {
680 	struct mail_transaction_log *log = view->index->log;
681 	struct mail_transaction_log_file *file, *prev_file;
682 	const char *reason;
683 	int ret;
684 
685 	if (log->files == NULL) {
686 		/* we shouldn't normally get here */
687 		return FALSE;
688 	}
689 	while (modseq < log->files->hdr.initial_modseq) {
690 		/* try to find the previous log file if it still exists */
691 		ret = mail_transaction_log_find_file(log,
692 			log->files->hdr.file_seq - 1, FALSE, &file, &reason);
693 		if (ret <= 0)
694 			return FALSE;
695 	}
696 
697 	prev_file = NULL;
698 	for (file = log->files; file != NULL; file = file->next) {
699 		if (modseq < file->hdr.initial_modseq)
700 			break;
701 		prev_file = file;
702 	}
703 
704 	if (prev_file == NULL) {
705 		/* the log file has been deleted already */
706 		return FALSE;
707 	}
708 
709 	*log_seq_r = prev_file->hdr.file_seq;
710 	if (mail_transaction_log_file_get_modseq_next_offset(prev_file, modseq,
711 							     log_offset_r) < 0)
712 		return FALSE;
713 
714 	if (*log_seq_r > view->log_file_head_seq ||
715 	    (*log_seq_r == view->log_file_head_seq &&
716 	     *log_offset_r > view->log_file_head_offset)) {
717 		/* modseq is already beyond our view. move it back so the
718 		   caller won't be confused. */
719 		*log_seq_r = view->log_file_head_seq;
720 		*log_offset_r = view->log_file_head_offset;
721 	}
722 	return TRUE;
723 }
724