1 /* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "ioloop.h"
5 #include "array.h"
6 #include "buffer.h"
7 #include "eacces-error.h"
8 #include "hash.h"
9 #include "str-sanitize.h"
10 #include "mmap-util.h"
11 #include "nfs-workarounds.h"
12 #include "read-full.h"
13 #include "write-full.h"
14 #include "mail-index-alloc-cache.h"
15 #include "mail-index-private.h"
16 #include "mail-index-view-private.h"
17 #include "mail-index-sync-private.h"
18 #include "mail-index-modseq.h"
19 #include "mail-transaction-log-private.h"
20 #include "mail-transaction-log-view-private.h"
21 #include "mail-cache.h"
22
23 #include <stdio.h>
24 #include <stddef.h>
25 #include <time.h>
26 #include <sys/stat.h>
27
28 struct mail_index_module_register mail_index_module_register = { 0 };
29
30 struct event_category event_category_mail_index = {
31 .name = "mail-index",
32 };
33
34 static void mail_index_close_nonopened(struct mail_index *index);
35
36 static const struct mail_index_optimization_settings default_optimization_set = {
37 .index = {
38 .rewrite_min_log_bytes = 8 * 1024,
39 .rewrite_max_log_bytes = 128 * 1024,
40 },
41 .log = {
42 .min_size = 32 * 1024,
43 .max_size = 1024 * 1024,
44 .min_age_secs = 5 * 60,
45 .log2_max_age_secs = 3600 * 24 * 2,
46 },
47 .cache = {
48 .unaccessed_field_drop_secs = 3600 * 24 * 30,
49 .record_max_size = 64 * 1024,
50 .max_size = 1024 * 1024 * 1024,
51 .purge_min_size = 32 * 1024,
52 .purge_delete_percentage = 20,
53 .purge_continued_percentage = 200,
54 .purge_header_continue_count = 4,
55 },
56 };
57
mail_index_alloc(struct event * parent_event,const char * dir,const char * prefix)58 struct mail_index *mail_index_alloc(struct event *parent_event,
59 const char *dir, const char *prefix)
60 {
61 struct mail_index *index;
62
63 index = i_new(struct mail_index, 1);
64 index->dir = i_strdup(dir);
65 index->prefix = i_strdup(prefix);
66 index->fd = -1;
67 index->event = event_create(parent_event);
68 event_add_category(index->event, &event_category_mail_index);
69
70 index->extension_pool =
71 pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);
72 p_array_init(&index->extensions, index->extension_pool, 5);
73 i_array_init(&index->module_contexts,
74 I_MIN(5, mail_index_module_register.id));
75
76 index->set.mode = 0600;
77 index->set.gid = (gid_t)-1;
78 index->set.lock_method = FILE_LOCK_METHOD_FCNTL;
79 index->set.max_lock_timeout_secs = UINT_MAX;
80 index->optimization_set = default_optimization_set;
81
82 index->keywords_ext_id =
83 mail_index_ext_register(index, MAIL_INDEX_EXT_KEYWORDS,
84 128, 2, 1);
85 index->keywords_pool = pool_alloconly_create("keywords", 512);
86 i_array_init(&index->keywords, 16);
87 hash_table_create(&index->keywords_hash, index->keywords_pool, 0,
88 strcase_hash, strcasecmp);
89 index->log = mail_transaction_log_alloc(index);
90 mail_index_modseq_init(index);
91 return index;
92 }
93
mail_index_free(struct mail_index ** _index)94 void mail_index_free(struct mail_index **_index)
95 {
96 struct mail_index *index = *_index;
97
98 *_index = NULL;
99
100 i_assert(index->open_count == 0);
101
102 mail_transaction_log_free(&index->log);
103 hash_table_destroy(&index->keywords_hash);
104 pool_unref(&index->extension_pool);
105 pool_unref(&index->keywords_pool);
106
107 array_free(&index->keywords);
108 array_free(&index->module_contexts);
109
110 event_unref(&index->event);
111 i_free(index->set.cache_dir);
112 i_free(index->set.ext_hdr_init_data);
113 i_free(index->set.gid_origin);
114 i_free(index->last_error.text);
115 i_free(index->dir);
116 i_free(index->prefix);
117 i_free(index->need_recreate);
118 i_free(index);
119 }
120
mail_index_set_cache_dir(struct mail_index * index,const char * dir)121 void mail_index_set_cache_dir(struct mail_index *index, const char *dir)
122 {
123 i_free(index->set.cache_dir);
124 index->set.cache_dir = i_strdup(dir);
125 }
126
mail_index_set_fsync_mode(struct mail_index * index,enum fsync_mode mode,enum mail_index_fsync_mask mask)127 void mail_index_set_fsync_mode(struct mail_index *index,
128 enum fsync_mode mode,
129 enum mail_index_fsync_mask mask)
130 {
131 index->set.fsync_mode = mode;
132 index->set.fsync_mask = mask;
133 }
134
mail_index_use_existing_permissions(struct mail_index * index)135 bool mail_index_use_existing_permissions(struct mail_index *index)
136 {
137 struct stat st;
138
139 if (MAIL_INDEX_IS_IN_MEMORY(index))
140 return FALSE;
141
142 if (stat(index->dir, &st) < 0) {
143 if (errno != ENOENT)
144 e_error(index->event, "stat(%s) failed: %m", index->dir);
145 return FALSE;
146 }
147
148 index->set.mode = st.st_mode & 0666;
149 if (S_ISDIR(st.st_mode) && (st.st_mode & S_ISGID) != 0) {
150 /* directory's GID is used automatically for new files */
151 index->set.gid = (gid_t)-1;
152 } else if ((st.st_mode & 0070) >> 3 == (st.st_mode & 0007)) {
153 /* group has same permissions as world, so don't bother
154 changing it */
155 index->set.gid = (gid_t)-1;
156 } else if (getegid() == st.st_gid) {
157 /* using our own gid, no need to change it */
158 index->set.gid = (gid_t)-1;
159 } else {
160 index->set.gid = st.st_gid;
161 }
162
163 i_free(index->set.gid_origin);
164 if (index->set.gid != (gid_t)-1)
165 index->set.gid_origin = i_strdup("preserved existing GID");
166 return TRUE;
167 }
168
mail_index_set_permissions(struct mail_index * index,mode_t mode,gid_t gid,const char * gid_origin)169 void mail_index_set_permissions(struct mail_index *index,
170 mode_t mode, gid_t gid, const char *gid_origin)
171 {
172 index->set.mode = mode & 0666;
173 index->set.gid = gid;
174
175 i_free(index->set.gid_origin);
176 index->set.gid_origin = i_strdup(gid_origin);
177 }
178
mail_index_set_lock_method(struct mail_index * index,enum file_lock_method lock_method,unsigned int max_timeout_secs)179 void mail_index_set_lock_method(struct mail_index *index,
180 enum file_lock_method lock_method,
181 unsigned int max_timeout_secs)
182 {
183 index->set.lock_method = lock_method;
184 index->set.max_lock_timeout_secs = max_timeout_secs;
185 }
186
mail_index_set_optimization_settings(struct mail_index * index,const struct mail_index_optimization_settings * set)187 void mail_index_set_optimization_settings(struct mail_index *index,
188 const struct mail_index_optimization_settings *set)
189 {
190 struct mail_index_optimization_settings *dest =
191 &index->optimization_set;
192
193 /* index */
194 if (set->index.rewrite_min_log_bytes != 0)
195 dest->index.rewrite_min_log_bytes = set->index.rewrite_min_log_bytes;
196 if (set->index.rewrite_max_log_bytes != 0)
197 dest->index.rewrite_max_log_bytes = set->index.rewrite_max_log_bytes;
198
199 /* log */
200 if (set->log.min_size != 0)
201 dest->log.min_size = set->log.min_size;
202 if (set->log.max_size != 0)
203 dest->log.max_size = set->log.max_size;
204 if (set->log.min_age_secs != 0)
205 dest->log.min_age_secs = set->log.min_age_secs;
206 if (set->log.log2_max_age_secs != 0)
207 dest->log.log2_max_age_secs = set->log.log2_max_age_secs;
208
209 /* cache */
210 if (set->cache.unaccessed_field_drop_secs != 0)
211 dest->cache.unaccessed_field_drop_secs =
212 set->cache.unaccessed_field_drop_secs;
213 if (set->cache.max_size != 0)
214 dest->cache.max_size = set->cache.max_size;
215 if (set->cache.purge_min_size != 0)
216 dest->cache.purge_min_size = set->cache.purge_min_size;
217 if (set->cache.purge_delete_percentage != 0)
218 dest->cache.purge_delete_percentage =
219 set->cache.purge_delete_percentage;
220 if (set->cache.purge_continued_percentage != 0)
221 dest->cache.purge_continued_percentage =
222 set->cache.purge_continued_percentage;
223 if (set->cache.purge_header_continue_count != 0)
224 dest->cache.purge_header_continue_count =
225 set->cache.purge_header_continue_count;
226 if (set->cache.record_max_size != 0)
227 dest->cache.record_max_size = set->cache.record_max_size;
228 }
229
mail_index_set_ext_init_data(struct mail_index * index,uint32_t ext_id,const void * data,size_t size)230 void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,
231 const void *data, size_t size)
232 {
233 const struct mail_index_registered_ext *rext;
234
235 i_assert(index->set.ext_hdr_init_data == NULL ||
236 index->set.ext_hdr_init_id == ext_id);
237
238 rext = array_idx(&index->extensions, ext_id);
239 i_assert(rext->hdr_size == size);
240
241 index->set.ext_hdr_init_id = ext_id;
242 i_free(index->set.ext_hdr_init_data);
243 index->set.ext_hdr_init_data = i_malloc(size);
244 memcpy(index->set.ext_hdr_init_data, data, size);
245 }
246
mail_index_ext_register(struct mail_index * index,const char * name,uint32_t default_hdr_size,uint16_t default_record_size,uint16_t default_record_align)247 uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
248 uint32_t default_hdr_size,
249 uint16_t default_record_size,
250 uint16_t default_record_align)
251 {
252 struct mail_index_registered_ext rext;
253 uint32_t ext_id;
254
255 if (*name == '\0' || strcmp(name, str_sanitize(name, SIZE_MAX)) != 0)
256 i_panic("mail_index_ext_register(%s): Invalid name", name);
257
258 if (default_record_size != 0 && default_record_align == 0) {
259 i_panic("mail_index_ext_register(%s): "
260 "Invalid record alignment", name);
261 }
262
263 if (mail_index_ext_lookup(index, name, &ext_id))
264 return ext_id;
265
266 i_zero(&rext);
267 rext.name = p_strdup(index->extension_pool, name);
268 rext.index_idx = array_count(&index->extensions);
269 rext.hdr_size = default_hdr_size;
270 rext.record_size = default_record_size;
271 rext.record_align = default_record_align;
272
273 array_push_back(&index->extensions, &rext);
274 return rext.index_idx;
275 }
276
mail_index_ext_register_resize_defaults(struct mail_index * index,uint32_t ext_id,uint32_t default_hdr_size,uint16_t default_record_size,uint16_t default_record_align)277 void mail_index_ext_register_resize_defaults(struct mail_index *index,
278 uint32_t ext_id,
279 uint32_t default_hdr_size,
280 uint16_t default_record_size,
281 uint16_t default_record_align)
282 {
283 struct mail_index_registered_ext *rext;
284
285 rext = array_idx_modifiable(&index->extensions, ext_id);
286 rext->hdr_size = default_hdr_size;
287 rext->record_size = default_record_size;
288 rext->record_align = default_record_align;
289 }
290
mail_index_ext_lookup(struct mail_index * index,const char * name,uint32_t * ext_id_r)291 bool mail_index_ext_lookup(struct mail_index *index, const char *name,
292 uint32_t *ext_id_r)
293 {
294 const struct mail_index_registered_ext *extensions;
295 unsigned int i, count;
296
297 extensions = array_get(&index->extensions, &count);
298 for (i = 0; i < count; i++) {
299 if (strcmp(extensions[i].name, name) == 0) {
300 *ext_id_r = i;
301 return TRUE;
302 }
303 }
304
305 *ext_id_r = (uint32_t)-1;
306 return FALSE;
307 }
308
mail_index_register_expunge_handler(struct mail_index * index,uint32_t ext_id,mail_index_expunge_handler_t * cb)309 void mail_index_register_expunge_handler(struct mail_index *index,
310 uint32_t ext_id,
311 mail_index_expunge_handler_t *cb)
312 {
313 struct mail_index_registered_ext *rext;
314
315 rext = array_idx_modifiable(&index->extensions, ext_id);
316 i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);
317
318 rext->expunge_handler = cb;
319 }
320
mail_index_unregister_expunge_handler(struct mail_index * index,uint32_t ext_id)321 void mail_index_unregister_expunge_handler(struct mail_index *index,
322 uint32_t ext_id)
323 {
324 struct mail_index_registered_ext *rext;
325
326 rext = array_idx_modifiable(&index->extensions, ext_id);
327 i_assert(rext->expunge_handler != NULL);
328
329 rext->expunge_handler = NULL;
330 }
331
mail_index_keyword_lookup(struct mail_index * index,const char * keyword,unsigned int * idx_r)332 bool mail_index_keyword_lookup(struct mail_index *index,
333 const char *keyword, unsigned int *idx_r)
334 {
335 char *key;
336 void *value;
337
338 /* keywords_hash keeps a name => index mapping of keywords.
339 Keywords are never removed from it, so the index values are valid
340 for the lifetime of the mail_index. */
341 if (hash_table_lookup_full(index->keywords_hash, keyword,
342 &key, &value)) {
343 *idx_r = POINTER_CAST_TO(value, unsigned int);
344 return TRUE;
345 }
346
347 *idx_r = UINT_MAX;
348 return FALSE;
349 }
350
mail_index_keyword_lookup_or_create(struct mail_index * index,const char * keyword,unsigned int * idx_r)351 void mail_index_keyword_lookup_or_create(struct mail_index *index,
352 const char *keyword,
353 unsigned int *idx_r)
354 {
355 char *keyword_dup;
356
357 i_assert(*keyword != '\0');
358
359 if (mail_index_keyword_lookup(index, keyword, idx_r))
360 return;
361
362 keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
363 *idx_r = array_count(&index->keywords);
364
365 hash_table_insert(index->keywords_hash, keyword_dup,
366 POINTER_CAST(*idx_r));
367 array_push_back(&index->keywords, &keyword);
368
369 /* keep the array NULL-terminated, but the NULL itself invisible */
370 array_append_zero(&index->keywords);
371 array_pop_back(&index->keywords);
372 }
373
ARRAY_TYPE(keywords)374 const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
375 {
376 return &index->keywords;
377 }
378
379 struct mail_keywords *
mail_index_keywords_create(struct mail_index * index,const char * const keywords[])380 mail_index_keywords_create(struct mail_index *index,
381 const char *const keywords[])
382 {
383 struct mail_keywords *k;
384 unsigned int src, dest, i, count;
385
386 count = str_array_length(keywords);
387 if (count == 0) {
388 k = i_new(struct mail_keywords, 1);
389 k->index = index;
390 k->refcount = 1;
391 return k;
392 }
393
394 /* @UNSAFE */
395 k = i_malloc(MALLOC_ADD(sizeof(struct mail_keywords),
396 MALLOC_MULTIPLY(sizeof(k->idx[0]), count)));
397 k->index = index;
398 k->refcount = 1;
399
400 /* look up the keywords from index. they're never removed from there
401 so we can permanently store indexes to them. */
402 for (src = dest = 0; src < count; src++) {
403 mail_index_keyword_lookup_or_create(index, keywords[src],
404 &k->idx[dest]);
405 /* ignore if this is a duplicate */
406 for (i = 0; i < src; i++) {
407 if (k->idx[i] == k->idx[dest])
408 break;
409 }
410 if (i == src)
411 dest++;
412 }
413 k->count = dest;
414 return k;
415 }
416
417 struct mail_keywords *
mail_index_keywords_create_from_indexes(struct mail_index * index,const ARRAY_TYPE (keyword_indexes)* keyword_indexes)418 mail_index_keywords_create_from_indexes(struct mail_index *index,
419 const ARRAY_TYPE(keyword_indexes)
420 *keyword_indexes)
421 {
422 struct mail_keywords *k;
423 const unsigned int *indexes;
424 unsigned int src, dest, i, count;
425
426 indexes = array_get(keyword_indexes, &count);
427 if (count == 0) {
428 k = i_new(struct mail_keywords, 1);
429 k->index = index;
430 k->refcount = 1;
431 return k;
432 }
433
434 /* @UNSAFE */
435 k = i_malloc(MALLOC_ADD(sizeof(struct mail_keywords),
436 MALLOC_MULTIPLY(sizeof(k->idx[0]), count)));
437 k->index = index;
438 k->refcount = 1;
439
440 /* copy but skip duplicates */
441 for (src = dest = 0; src < count; src++) {
442 for (i = 0; i < src; i++) {
443 if (k->idx[i] == indexes[src])
444 break;
445 }
446 if (i == src)
447 k->idx[dest++] = indexes[src];
448 }
449 k->count = dest;
450 return k;
451 }
452
mail_index_keywords_ref(struct mail_keywords * keywords)453 void mail_index_keywords_ref(struct mail_keywords *keywords)
454 {
455 keywords->refcount++;
456 }
457
mail_index_keywords_unref(struct mail_keywords ** _keywords)458 void mail_index_keywords_unref(struct mail_keywords **_keywords)
459 {
460 struct mail_keywords *keywords = *_keywords;
461
462 i_assert(keywords->refcount > 0);
463
464 *_keywords = NULL;
465 if (--keywords->refcount == 0)
466 i_free(keywords);
467 }
468
mail_index_try_open_only(struct mail_index * index)469 int mail_index_try_open_only(struct mail_index *index)
470 {
471 i_assert(index->fd == -1);
472 i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
473
474 /* Note that our caller must close index->fd by itself. */
475 if (index->readonly)
476 errno = EACCES;
477 else {
478 index->fd = nfs_safe_open(index->filepath, O_RDWR);
479 index->readonly = FALSE;
480 }
481
482 if (index->fd == -1 && errno == EACCES) {
483 index->fd = open(index->filepath, O_RDONLY);
484 index->readonly = TRUE;
485 }
486
487 if (index->fd == -1) {
488 if (errno != ENOENT) {
489 mail_index_set_syscall_error(index, "open()");
490 return -1;
491 }
492
493 /* have to create it */
494 return 0;
495 }
496 return 1;
497 }
498
499 static int
mail_index_try_open(struct mail_index * index)500 mail_index_try_open(struct mail_index *index)
501 {
502 int ret;
503
504 i_assert(index->fd == -1);
505
506 if (MAIL_INDEX_IS_IN_MEMORY(index))
507 return 0;
508
509 ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
510 if (ret == 0 && !index->readonly) {
511 /* it's corrupted - recreate it */
512 if (index->fd != -1) {
513 if (close(index->fd) < 0)
514 mail_index_set_syscall_error(index, "close()");
515 index->fd = -1;
516 }
517 }
518 return ret;
519 }
520
mail_index_create_tmp_file(struct mail_index * index,const char * path_prefix,const char ** path_r)521 int mail_index_create_tmp_file(struct mail_index *index,
522 const char *path_prefix, const char **path_r)
523 {
524 mode_t old_mask;
525 const char *path;
526 int fd;
527
528 i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
529
530 path = *path_r = t_strconcat(path_prefix, ".tmp", NULL);
531 old_mask = umask(0);
532 fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->set.mode);
533 umask(old_mask);
534 if (fd == -1 && errno == EEXIST) {
535 /* stale temp file. unlink and recreate rather than overwriting,
536 just to make sure locking problems won't cause corruption */
537 if (i_unlink(path) < 0)
538 return -1;
539 old_mask = umask(0);
540 fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->set.mode);
541 umask(old_mask);
542 }
543 if (fd == -1) {
544 mail_index_file_set_syscall_error(index, path, "creat()");
545 return -1;
546 }
547
548 mail_index_fchown(index, fd, path);
549 return fd;
550 }
551
mail_index_get_cache_path(struct mail_index * index)552 static const char *mail_index_get_cache_path(struct mail_index *index)
553 {
554 const char *dir;
555
556 if (index->set.cache_dir != NULL)
557 dir = index->set.cache_dir;
558 else if (index->dir != NULL)
559 dir = index->dir;
560 else
561 return NULL;
562 return t_strconcat(dir, "/", index->prefix,
563 MAIL_CACHE_FILE_SUFFIX, NULL);
564 }
565
mail_index_open_files(struct mail_index * index,enum mail_index_open_flags flags)566 static int mail_index_open_files(struct mail_index *index,
567 enum mail_index_open_flags flags)
568 {
569 int ret;
570
571 ret = mail_transaction_log_open(index->log);
572 if (ret == 0) {
573 if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
574 return 0;
575
576 /* if dovecot.index exists, read it first so that we can get
577 the correct indexid and log sequence */
578 (void)mail_index_try_open(index);
579
580 if (index->indexid == 0) {
581 /* Create a new indexid for us. If we're opening index
582 into memory, index->map doesn't exist yet. */
583 index->indexid = ioloop_time;
584 index->initial_create = TRUE;
585 if (index->map != NULL)
586 index->map->hdr.indexid = index->indexid;
587 }
588
589 ret = mail_transaction_log_create(index->log, FALSE);
590 if (index->map != NULL) {
591 /* log creation could have changed it if someone else
592 just created it. */
593 index->map->hdr.indexid = index->indexid;
594 }
595 index->initial_create = FALSE;
596 }
597 if (ret >= 0) {
598 ret = index->map != NULL ? 1 : mail_index_try_open(index);
599 if (ret == 0 && !index->readonly) {
600 /* corrupted */
601 mail_transaction_log_close(index->log);
602 ret = mail_transaction_log_create(index->log, TRUE);
603 if (ret == 0) {
604 if (index->map != NULL)
605 mail_index_unmap(&index->map);
606 index->map = mail_index_map_alloc(index);
607 }
608 }
609 }
610 if (ret < 0) {
611 /* open/create failed, fallback to in-memory indexes */
612 if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
613 return -1;
614
615 if (mail_index_move_to_memory(index) < 0)
616 return -1;
617 }
618
619 if (index->cache == NULL) {
620 const char *path = mail_index_get_cache_path(index);
621 index->cache = mail_cache_open_or_create_path(index, path);
622 }
623 return 1;
624 }
625
626 static int
mail_index_open_opened(struct mail_index * index,enum mail_index_open_flags flags)627 mail_index_open_opened(struct mail_index *index,
628 enum mail_index_open_flags flags)
629 {
630 int ret;
631
632 i_assert(index->map != NULL);
633
634 if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
635 /* index was marked corrupted. we'll probably need to
636 recreate the files. */
637 mail_index_unmap(&index->map);
638 mail_index_close_file(index);
639 mail_transaction_log_close(index->log);
640 if ((ret = mail_index_open_files(index, flags)) <= 0)
641 return ret;
642 }
643
644 index->open_count++;
645 return 1;
646 }
647
mail_index_open(struct mail_index * index,enum mail_index_open_flags flags)648 int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags)
649 {
650 int ret;
651
652 if (index->open_count > 0) {
653 if ((ret = mail_index_open_opened(index, flags)) <= 0) {
654 /* doesn't exist and create flag not used */
655 }
656 return ret;
657 }
658
659 index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
660 i_strdup("(in-memory index)") :
661 i_strconcat(index->dir, "/", index->prefix, NULL);
662
663 mail_index_reset_error(index);
664 index->readonly = FALSE;
665 index->log_sync_locked = FALSE;
666 index->flags = flags;
667 index->readonly = (flags & MAIL_INDEX_OPEN_FLAG_READONLY) != 0;
668 if ((flags & MAIL_INDEX_OPEN_FLAG_DEBUG) != 0)
669 event_set_forced_debug(index->event, TRUE);
670 else
671 event_unset_forced_debug(index->event);
672
673 if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 &&
674 index->set.fsync_mode != FSYNC_MODE_ALWAYS)
675 i_fatal("nfs flush requires mail_fsync=always");
676 if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 &&
677 (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
678 i_fatal("nfs flush requires mmap_disable=yes");
679
680 /* NOTE: increase open_count only after mail_index_open_files().
681 it's used elsewhere to check if we're doing an initial opening
682 of the index files */
683 if ((ret = mail_index_open_files(index, flags)) <= 0) {
684 /* doesn't exist and create flag not used */
685 mail_index_close_nonopened(index);
686 return ret;
687 }
688
689 index->open_count++;
690
691 if (index->log->head == NULL) {
692 mail_index_close(index);
693 mail_index_set_error(index, "Index is corrupted "
694 "(log->view->head == NULL)");
695 return -1;
696 }
697
698 i_assert(index->map != NULL);
699 mail_index_alloc_cache_index_opened(index);
700 return 1;
701 }
702
mail_index_open_or_create(struct mail_index * index,enum mail_index_open_flags flags)703 int mail_index_open_or_create(struct mail_index *index,
704 enum mail_index_open_flags flags)
705 {
706 int ret;
707
708 flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
709 ret = mail_index_open(index, flags);
710 i_assert(ret != 0);
711 return ret < 0 ? -1 : 0;
712 }
713
mail_index_close_file(struct mail_index * index)714 void mail_index_close_file(struct mail_index *index)
715 {
716 if (index->fd != -1) {
717 if (close(index->fd) < 0)
718 mail_index_set_syscall_error(index, "close()");
719 index->fd = -1;
720 }
721 }
722
mail_index_close_nonopened(struct mail_index * index)723 static void mail_index_close_nonopened(struct mail_index *index)
724 {
725 i_assert(!index->syncing);
726
727 if (index->views != NULL) {
728 i_panic("Leaked view for index %s: Opened in %s:%u",
729 index->filepath, index->views->source_filename,
730 index->views->source_linenum);
731 }
732 i_assert(index->views == NULL);
733
734 if (index->map != NULL)
735 mail_index_unmap(&index->map);
736
737 mail_index_close_file(index);
738 mail_transaction_log_close(index->log);
739 if (index->cache != NULL)
740 mail_cache_free(&index->cache);
741
742 i_free_and_null(index->filepath);
743
744 index->indexid = 0;
745 }
746
mail_index_close(struct mail_index * index)747 void mail_index_close(struct mail_index *index)
748 {
749 i_assert(index->open_count > 0);
750
751 mail_index_alloc_cache_index_closing(index);
752 if (--index->open_count == 0)
753 mail_index_close_nonopened(index);
754 }
755
mail_index_unlink(struct mail_index * index)756 int mail_index_unlink(struct mail_index *index)
757 {
758 const char *path;
759 int last_errno = 0;
760
761 if (MAIL_INDEX_IS_IN_MEMORY(index) || index->readonly)
762 return 0;
763
764 /* main index */
765 if (unlink(index->filepath) < 0 && errno != ENOENT)
766 last_errno = errno;
767
768 /* logs */
769 path = t_strconcat(index->filepath, MAIL_TRANSACTION_LOG_SUFFIX, NULL);
770 if (unlink(path) < 0 && errno != ENOENT)
771 last_errno = errno;
772
773 path = t_strconcat(index->filepath,
774 MAIL_TRANSACTION_LOG_SUFFIX".2", NULL);
775 if (unlink(path) < 0 && errno != ENOENT)
776 last_errno = errno;
777
778 /* cache */
779 path = t_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
780 if (unlink(path) < 0 && errno != ENOENT)
781 last_errno = errno;
782
783 if (last_errno == 0)
784 return 0;
785 else {
786 errno = last_errno;
787 return -1;
788 }
789 }
790
mail_index_reopen_if_changed(struct mail_index * index,bool * reopened_r,const char ** reason_r)791 int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r,
792 const char **reason_r)
793 {
794 struct stat st1, st2;
795 int ret;
796
797 *reopened_r = FALSE;
798
799 if (MAIL_INDEX_IS_IN_MEMORY(index)) {
800 *reason_r = "in-memory index";
801 return 0;
802 }
803
804 if (index->fd == -1)
805 goto final;
806
807 if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0)
808 nfs_flush_file_handle_cache(index->filepath);
809 if (nfs_safe_stat(index->filepath, &st2) < 0) {
810 if (errno == ENOENT) {
811 *reason_r = "index not found via stat()";
812 return 0;
813 }
814 mail_index_set_syscall_error(index, "stat()");
815 return -1;
816 }
817
818 if (fstat(index->fd, &st1) < 0) {
819 if (!ESTALE_FSTAT(errno)) {
820 mail_index_set_syscall_error(index, "fstat()");
821 return -1;
822 }
823 /* deleted/recreated, reopen */
824 *reason_r = "index is stale";
825 } else if (st1.st_ino == st2.st_ino &&
826 CMP_DEV_T(st1.st_dev, st2.st_dev)) {
827 /* the same file */
828 *reason_r = "index unchanged";
829 return 1;
830 } else {
831 *reason_r = "index inode changed";
832 }
833
834 /* new file, new locks. the old fd can keep its locks, they don't
835 matter anymore as no-one's going to modify the file. */
836 mail_index_close_file(index);
837
838 final:
839 if ((ret = mail_index_try_open_only(index)) == 0)
840 *reason_r = "index not found via open()";
841 else if (ret > 0) {
842 *reason_r = "index opened";
843 *reopened_r = TRUE;
844 }
845 return ret;
846 }
847
mail_index_refresh(struct mail_index * index)848 int mail_index_refresh(struct mail_index *index)
849 {
850 int ret;
851
852 ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
853 return ret <= 0 ? -1 : 0;
854 }
855
mail_index_get_cache(struct mail_index * index)856 struct mail_cache *mail_index_get_cache(struct mail_index *index)
857 {
858 return index->cache;
859 }
860
mail_index_set_error(struct mail_index * index,const char * fmt,...)861 void mail_index_set_error(struct mail_index *index, const char *fmt, ...)
862 {
863 va_list va;
864
865 i_free(index->last_error.text);
866
867 if (fmt == NULL)
868 index->last_error.text = NULL;
869 else {
870 va_start(va, fmt);
871 index->last_error.text = i_strdup_vprintf(fmt, va);
872 va_end(va);
873
874 e_error(index->event, "%s", index->last_error.text);
875 }
876 }
877
mail_index_set_error_nolog(struct mail_index * index,const char * str)878 void mail_index_set_error_nolog(struct mail_index *index, const char *str)
879 {
880 i_assert(str != NULL);
881
882 char *old_error = index->last_error.text;
883 index->last_error.text = i_strdup(str);
884 i_free(old_error);
885 }
886
mail_index_is_in_memory(struct mail_index * index)887 bool mail_index_is_in_memory(struct mail_index *index)
888 {
889 return MAIL_INDEX_IS_IN_MEMORY(index);
890 }
891
mail_index_set_as_in_memory(struct mail_index * index)892 static void mail_index_set_as_in_memory(struct mail_index *index)
893 {
894 i_free_and_null(index->dir);
895
896 i_free(index->filepath);
897 index->filepath = i_strdup("(in-memory index)");
898 }
899
mail_index_move_to_memory(struct mail_index * index)900 int mail_index_move_to_memory(struct mail_index *index)
901 {
902 struct mail_index_map *map;
903
904 if (MAIL_INDEX_IS_IN_MEMORY(index))
905 return index->map == NULL ? -1 : 0;
906
907 if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0)
908 return -1;
909
910 if (index->map == NULL) {
911 /* index was never even opened. just mark it as being in
912 memory and let the caller re-open the index. */
913 i_assert(index->fd == -1);
914 mail_index_set_as_in_memory(index);
915 return -1;
916 }
917
918 /* move index map to memory */
919 if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
920 map = mail_index_map_clone(index->map);
921 mail_index_unmap(&index->map);
922 index->map = map;
923 }
924
925 if (index->log != NULL) {
926 /* move transaction log to memory */
927 if (mail_transaction_log_move_to_memory(index->log) < 0)
928 return -1;
929 }
930
931 if (index->fd != -1) {
932 if (close(index->fd) < 0)
933 mail_index_set_syscall_error(index, "close()");
934 index->fd = -1;
935 }
936 mail_index_set_as_in_memory(index);
937 return 0;
938 }
939
mail_index_mark_corrupted(struct mail_index * index)940 void mail_index_mark_corrupted(struct mail_index *index)
941 {
942 index->indexid = 0;
943
944 index->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
945 if (!index->readonly) {
946 if (unlink(index->filepath) < 0 &&
947 errno != ENOENT && errno != ESTALE)
948 mail_index_set_syscall_error(index, "unlink()");
949 (void)mail_transaction_log_unlink(index->log);
950 }
951 }
952
mail_index_is_deleted(struct mail_index * index)953 bool mail_index_is_deleted(struct mail_index *index)
954 {
955 return index->index_delete_requested || index->index_deleted;
956 }
957
mail_index_get_modification_time(struct mail_index * index,time_t * mtime_r)958 int mail_index_get_modification_time(struct mail_index *index, time_t *mtime_r)
959 {
960 struct stat st;
961 const char *path;
962
963 *mtime_r = 0;
964 if (MAIL_INDEX_IS_IN_MEMORY(index)) {
965 /* this function doesn't make sense for in-memory indexes */
966 return 0;
967 }
968
969 /* index may not be open, so index->filepath may be NULL */
970 path = t_strconcat(index->dir, "/", index->prefix,
971 MAIL_TRANSACTION_LOG_SUFFIX, NULL);
972 if (stat(path, &st) < 0) {
973 if (errno == ENOENT) {
974 /* .log is always supposed to exist - don't bother
975 trying to stat(dovecot.index) */
976 return 0;
977 }
978 mail_index_file_set_syscall_error(index, path, "stat()");
979 return -1;
980 }
981 *mtime_r = st.st_mtime;
982 return 0;
983 }
984
mail_index_fchown(struct mail_index * index,int fd,const char * path)985 void mail_index_fchown(struct mail_index *index, int fd, const char *path)
986 {
987 mode_t mode;
988
989 if (index->set.gid == (gid_t)-1) {
990 /* no gid changing */
991 return;
992 } else if (fchown(fd, (uid_t)-1, index->set.gid) == 0) {
993 /* success */
994 return;
995 } if ((index->set.mode & 0060) >> 3 == (index->set.mode & 0006)) {
996 /* group and world permissions are the same, so group doesn't
997 really matter. ignore silently. */
998 return;
999 }
1000 if (errno != EPERM)
1001 mail_index_file_set_syscall_error(index, path, "fchown()");
1002 else {
1003 mail_index_set_error(index, "%s",
1004 eperm_error_get_chgrp("fchown", path, index->set.gid,
1005 index->set.gid_origin));
1006 }
1007
1008 /* continue, but change permissions so that only the common
1009 subset of group and world is used. this makes sure no one
1010 gets any extra permissions. */
1011 mode = ((index->set.mode & 0060) >> 3) & (index->set.mode & 0006);
1012 mode |= (mode << 3) | (index->set.mode & 0600);
1013 if (fchmod(fd, mode) < 0)
1014 mail_index_file_set_syscall_error(index, path, "fchmod()");
1015 }
1016
mail_index_lock_sync(struct mail_index * index,const char * lock_reason)1017 int mail_index_lock_sync(struct mail_index *index, const char *lock_reason)
1018 {
1019 uint32_t file_seq;
1020 uoff_t file_offset;
1021
1022 return mail_transaction_log_sync_lock(index->log, lock_reason,
1023 &file_seq, &file_offset);
1024 }
1025
mail_index_unlock(struct mail_index * index,const char * long_lock_reason)1026 void mail_index_unlock(struct mail_index *index, const char *long_lock_reason)
1027 {
1028 mail_transaction_log_sync_unlock(index->log, long_lock_reason);
1029 }
1030
mail_index_is_locked(struct mail_index * index)1031 bool mail_index_is_locked(struct mail_index *index)
1032 {
1033 return index->log_sync_locked;
1034 }
1035
mail_index_set_syscall_error(struct mail_index * index,const char * function)1036 void mail_index_set_syscall_error(struct mail_index *index,
1037 const char *function)
1038 {
1039 mail_index_file_set_syscall_error(index, index->filepath, function);
1040 }
1041
mail_index_file_set_syscall_error(struct mail_index * index,const char * filepath,const char * function)1042 void mail_index_file_set_syscall_error(struct mail_index *index,
1043 const char *filepath,
1044 const char *function)
1045 {
1046 const char *errstr;
1047
1048 i_assert(filepath != NULL);
1049 i_assert(function != NULL);
1050
1051 if (errno == ENOENT) {
1052 struct stat st;
1053 int old_errno = errno;
1054 i_assert(index->log->filepath != NULL);
1055 if (nfs_safe_stat(index->log->filepath, &st) < 0 &&
1056 errno == ENOENT) {
1057 /* the index log has gone away */
1058 index->index_deleted = TRUE;
1059 errno = old_errno;
1060 return;
1061 }
1062 errno = old_errno;
1063 }
1064
1065 if (ENOSPACE(errno)) {
1066 index->last_error.nodiskspace = TRUE;
1067 if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) == 0)
1068 return;
1069 }
1070
1071 if (errno == EACCES) {
1072 function = t_strcut(function, '(');
1073 if (strcmp(function, "creat") == 0 ||
1074 str_begins(function, "file_dotlock_"))
1075 errstr = eacces_error_get_creating(function, filepath);
1076 else
1077 errstr = eacces_error_get(function, filepath);
1078 mail_index_set_error(index, "%s", errstr);
1079 } else {
1080 const char *suffix = errno != EFBIG ? "" :
1081 " (process was started with ulimit -f limit)";
1082 mail_index_set_error(index, "%s failed with file %s: "
1083 "%m%s", function, filepath, suffix);
1084 }
1085 }
1086
mail_index_get_error_message(struct mail_index * index)1087 const char *mail_index_get_error_message(struct mail_index *index)
1088 {
1089 return index->last_error.text;
1090 }
1091
mail_index_reset_error(struct mail_index * index)1092 void mail_index_reset_error(struct mail_index *index)
1093 {
1094 i_free(index->last_error.text);
1095 i_zero(&index->last_error);
1096 }
1097