1 #ifndef MAIL_INDEX_PRIVATE_H 2 #define MAIL_INDEX_PRIVATE_H 3 4 #include "file-lock.h" 5 #include "mail-index.h" 6 #include "mail-index-util.h" 7 #include "mail-index-view-private.h" 8 #include "mail-index-transaction-private.h" 9 10 #include <sys/stat.h> 11 12 struct mail_transaction_header; 13 struct mail_transaction_log_view; 14 struct mail_index_sync_map_ctx; 15 16 /* How large index files to mmap() instead of reading to memory. */ 17 #define MAIL_INDEX_MMAP_MIN_SIZE (1024*64) 18 /* How many times to retry opening index files if read/fstat returns ESTALE. 19 This happens with NFS when the file has been deleted (ie. index file was 20 rewritten by another computer than us). */ 21 #define MAIL_INDEX_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT 22 /* Large extension header sizes are probably caused by file corruption, so 23 try to catch them by limiting the header size. */ 24 #define MAIL_INDEX_EXT_HEADER_MAX_SIZE (1024*1024*16-1) 25 26 #define MAIL_INDEX_IS_IN_MEMORY(index) \ 27 ((index)->dir == NULL) 28 29 #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \ 30 ((map)->rec_map->mmap_base == NULL) 31 32 #define MAIL_INDEX_MAP_IDX(map, idx) \ 33 ((struct mail_index_record *) \ 34 PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size)) 35 #define MAIL_INDEX_REC_AT_SEQ(map, seq) \ 36 ((struct mail_index_record *) \ 37 PTR_OFFSET((map)->rec_map->records, ((seq)-1) * (map)->hdr.record_size)) 38 39 #define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \ 40 ((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \ 41 (u)->modseq_inc_flag == 0) 42 43 #define MAIL_INDEX_EXT_KEYWORDS "keywords" 44 45 typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx, 46 const void *data, void **sync_context); 47 48 #define MAIL_INDEX_HEADER_SIZE_ALIGN(size) \ 49 (((size) + 7) & ~7U) 50 51 /* In-memory copy of struct mail_index_ext_header */ 52 struct mail_index_ext { 53 const char *name; 54 uint32_t index_idx; /* index ext_id */ 55 uint32_t reset_id; 56 uint32_t ext_offset; /* points to beginning of mail_index_ext_header */ 57 uint32_t hdr_offset; /* points to mail_index_ext_header.data[] */ 58 uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ 59 uint16_t record_offset; 60 uint16_t record_size; 61 uint16_t record_align; 62 }; 63 64 struct mail_index_ext_header { 65 /* Size of data[], i.e. the extension size in header */ 66 uint32_t hdr_size; 67 /* If reset_id changes, all of the extension record data is 68 invalidated. For example with cache files reset_id must match the 69 cache header's file_seq or the cache offsets aren't valid. */ 70 uint32_t reset_id; 71 /* Offset of this extension in struct mail_index_record. */ 72 uint16_t record_offset; 73 /* Size of this extension in struct mail_index_record. */ 74 uint16_t record_size; 75 /* Required alignment of this extension in struct mail_index_record. 76 It's expected that record_offset is correctly aligned. This is used 77 only when rearranging fields due to adding/removing other 78 extensions. */ 79 uint16_t record_align; 80 /* Size of name[], which contains the extension's unique name. */ 81 uint16_t name_size; 82 /* unsigned char name[name_size]; */ 83 /* Extension header data, if any. This starts from the next 64-bit 84 aligned offset after name[]. */ 85 /* unsigned char data[hdr_size]; */ 86 }; 87 88 struct mail_index_keyword_header { 89 uint32_t keywords_count; 90 /* struct mail_index_keyword_header_rec[] */ 91 /* char name[][] */ 92 }; 93 94 struct mail_index_keyword_header_rec { 95 uint32_t unused; /* for backwards compatibility */ 96 uint32_t name_offset; /* relative to beginning of name[] */ 97 }; 98 99 enum mail_index_sync_handler_type { 100 MAIL_INDEX_SYNC_HANDLER_FILE = 0x01, 101 MAIL_INDEX_SYNC_HANDLER_HEAD = 0x02, 102 MAIL_INDEX_SYNC_HANDLER_VIEW = 0x04 103 }; 104 105 struct mail_index_registered_ext { 106 const char *name; 107 uint32_t index_idx; /* index ext_id */ 108 uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ 109 uint16_t record_size; 110 uint16_t record_align; 111 112 mail_index_expunge_handler_t *expunge_handler; 113 }; 114 115 struct mail_index_record_map { 116 ARRAY(struct mail_index_map *) maps; 117 118 void *mmap_base; 119 size_t mmap_size, mmap_used_size; 120 121 buffer_t *buffer; 122 123 void *records; /* struct mail_index_record[] */ 124 unsigned int records_count; 125 126 struct mail_index_map_modseq *modseq; 127 uint32_t last_appended_uid; 128 }; 129 130 #define MAIL_INDEX_MAP_HDR_OFFSET(map, hdr_offset) \ 131 CONST_PTR_OFFSET((map)->hdr_copy_buf->data, hdr_offset) 132 struct mail_index_map { 133 struct mail_index *index; 134 int refcount; 135 136 /* Copy of the base header for convenience. Note that base_header_size 137 may be smaller or larger than this struct. If it's smaller, the last 138 fields in the struct are filled with zeroes. */ 139 struct mail_index_header hdr; 140 /* Copy of the full header. */ 141 buffer_t *hdr_copy_buf; 142 143 pool_t extension_pool; 144 ARRAY(struct mail_index_ext) extensions; 145 ARRAY(uint32_t) ext_id_map; /* index -> file */ 146 147 ARRAY(unsigned int) keyword_idx_map; /* file -> index */ 148 149 struct mail_index_record_map *rec_map; 150 }; 151 152 struct mail_index_module_register { 153 unsigned int id; 154 }; 155 156 union mail_index_module_context { 157 struct mail_index_module_register *reg; 158 }; 159 160 struct mail_index_settings { 161 /* Directory path for .cache file. Set via 162 mail_index_set_cache_dir(). */ 163 char *cache_dir; 164 165 /* fsyncing behavior. Set via mail_index_set_fsync_mode(). */ 166 enum fsync_mode fsync_mode; 167 enum mail_index_fsync_mask fsync_mask; 168 169 /* Index file permissions. Set via mail_index_set_permissions(). */ 170 mode_t mode; 171 gid_t gid; 172 char *gid_origin; 173 174 /* Lock settings. Set via mail_index_set_lock_method(). */ 175 enum file_lock_method lock_method; 176 unsigned int max_lock_timeout_secs; 177 178 /* Initial extension added to newly created indexes. Set via 179 mail_index_set_ext_init_data(). */ 180 uint32_t ext_hdr_init_id; 181 void *ext_hdr_init_data; 182 }; 183 184 struct mail_index_error { 185 /* Human-readable error text */ 186 char *text; 187 188 /* Error happened because there's no disk space, i.e. syscall failed 189 with ENOSPC or EDQUOT. */ 190 bool nodiskspace:1; 191 }; 192 193 struct mail_index { 194 /* Directory path for the index, or NULL for in-memory indexes. */ 195 char *dir; 196 /* Filename prefix for the index, e.g. "dovecot.index." */ 197 char *prefix; 198 struct event *event; 199 enum mail_index_open_flags flags; 200 struct mail_index_settings set; 201 struct mail_index_optimization_settings optimization_set; 202 203 struct mail_cache *cache; 204 struct mail_transaction_log *log; 205 206 char *filepath; 207 int fd; 208 /* Linked list of currently opened views */ 209 struct mail_index_view *views; 210 /* Latest map */ 211 struct mail_index_map *map; 212 213 /* ID number that permanently identifies the index. This is stored in 214 the index files' headers. If the indexids suddenly changes, it means 215 that the index has been completely recreated and needs to be 216 reopened (e.g. the mailbox was deleted and recreated while it 217 was open). */ 218 uint32_t indexid; 219 /* Views initially use this same ID value. This ID is incremented 220 whenever something unexpected happens to the index that prevents 221 syncing existing views. When the view's inconsistency_id doesn't 222 match this one, the view is marked as inconsistent. */ 223 unsigned int inconsistency_id; 224 /* How many times this index has been opened with mail_index_open(). */ 225 unsigned int open_count; 226 227 /* These contain the log_file_seq and log_file_tail_offset that exists 228 in dovecot.index file's header. These are used to figure out if it's 229 time to rewrite the dovecot.index file. Note that these aren't 230 available in index->map->hdr, because it gets updated when 231 transaction log file is read. */ 232 uint32_t main_index_hdr_log_file_seq; 233 uint32_t main_index_hdr_log_file_tail_offset; 234 235 /* log file which last updated index_deleted */ 236 uint32_t index_delete_changed_file_seq; 237 238 /* transaction log head seq/offset when we last fscked */ 239 uint32_t fsck_log_head_file_seq; 240 uoff_t fsck_log_head_file_offset; 241 242 /* syncing will update this if non-NULL */ 243 struct mail_index_transaction_commit_result *sync_commit_result; 244 /* Delayed log2_rotate_time update to mail_index_header. This is set 245 and unset within the same sync. */ 246 uint32_t hdr_log2_rotate_time_delayed_update; 247 248 /* Registered extensions */ 249 pool_t extension_pool; 250 ARRAY(struct mail_index_registered_ext) extensions; 251 252 /* All keywords that have ever been used in this index. Keywords are 253 only added here, never removed. */ 254 pool_t keywords_pool; 255 ARRAY_TYPE(keywords) keywords; 256 HASH_TABLE(char *, void *) keywords_hash; /* name -> unsigned int idx */ 257 258 /* Registered extension IDs */ 259 uint32_t keywords_ext_id; 260 uint32_t modseq_ext_id; 261 262 /* Module-specific contexts. */ 263 ARRAY(union mail_index_module_context *) module_contexts; 264 265 /* Last error returned by mail_index_get_error_message(). 266 Cleared by mail_index_reset_error(). */ 267 struct mail_index_error last_error; 268 /* Timestamp when mmap() failure was logged the last time. This is used 269 to prevent logging the same error too rapidly. This could happen 270 e.g. if mmap()ing a large cache file that exceeeds process's 271 VSZ limit. */ 272 time_t last_mmap_error_time; 273 /* If non-NULL, dovecot.index should be recreated as soon as possible. 274 The reason for why the recreation is wanted is stored as human- 275 readable text. */ 276 char *need_recreate; 277 278 /* Mapping has noticed non-external MAIL_TRANSACTION_INDEX_DELETED 279 record, i.e. a request to mark the index deleted. The next sync 280 will finish the deletion by writing external 281 MAIL_TRANSACTION_INDEX_DELETED record. */ 282 bool index_delete_requested:1; 283 /* Mapping has noticed external MAIL_TRANSACTION_INDEX_DELETED record, 284 or index was unexpectedly deleted under us. No more changes are 285 allowed to the index, except undeletion. */ 286 bool index_deleted:1; 287 /* .log is locked for syncing. This is the main exclusive lock for 288 indexes. */ 289 bool log_sync_locked:1; 290 /* Main index or .log couldn't be opened read-write */ 291 bool readonly:1; 292 /* mail_index_map() is running */ 293 bool mapping:1; 294 /* mail_index_sync_*() is running */ 295 bool syncing:1; 296 /* Mapping has read more from .log than it preferred. Use 297 mail_index_base_optimization_settings.rewrite_min_log_bytes the next 298 time when checking if index needs a rewrite. */ 299 bool index_min_write:1; 300 /* mail_index_modseq_enable() has been called. Track per-flag 301 modseq numbers in memory (global modseqs are tracked anyway). */ 302 bool modseqs_enabled:1; 303 /* mail_index_open() is creating new index files */ 304 bool initial_create:1; 305 /* TRUE after mail_index_map() has succeeded */ 306 bool initial_mapped:1; 307 /* The next mail_index_map() must reopen the main index, because the 308 currently opened one is too old. */ 309 bool reopen_main_index:1; 310 /* Index has been fsck'd, but mail_index_reset_fscked() hasn't been 311 called yet. */ 312 bool fscked:1; 313 }; 314 315 extern struct mail_index_module_register mail_index_module_register; 316 extern struct event_category event_category_mail_index; 317 318 /* Add/replace expunge handler for specified extension. */ 319 void mail_index_register_expunge_handler(struct mail_index *index, 320 uint32_t ext_id, 321 mail_index_expunge_handler_t *callback); 322 void mail_index_unregister_expunge_handler(struct mail_index *index, 323 uint32_t ext_id); 324 325 int mail_index_create_tmp_file(struct mail_index *index, 326 const char *path_prefix, const char **path_r); 327 328 int mail_index_try_open_only(struct mail_index *index); 329 void mail_index_close_file(struct mail_index *index); 330 /* Returns 1 if index was successfully (re-)opened, 0 if the index no longer 331 exists, -1 if I/O error. If 1 is returned, reopened_r=TRUE if a new index 332 was actually reopened (or if index wasn't even open before this call). */ 333 int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r, 334 const char **reason_r); 335 /* Update/rewrite the main index file from index->map */ 336 void mail_index_write(struct mail_index *index, bool want_rotate, 337 const char *reason); 338 339 void mail_index_flush_read_cache(struct mail_index *index, const char *path, 340 int fd, bool locked); 341 342 int mail_index_lock_fd(struct mail_index *index, const char *path, int fd, 343 int lock_type, unsigned int timeout_secs, 344 struct file_lock **lock_r); 345 346 /* Allocate a new empty map. */ 347 struct mail_index_map *mail_index_map_alloc(struct mail_index *index); 348 /* Replace index->map with the latest index changes. This may reopen the index 349 file and/or it may read the latest changes from transaction log. The log is 350 read up to EOF, but non-synced expunges are skipped. 351 352 If we mmap()ed the index file, the map is returned locked. 353 354 Returns 1 = ok, 0 = corrupted, -1 = error. */ 355 int mail_index_map(struct mail_index *index, 356 enum mail_index_sync_handler_type type); 357 /* Unreference given mapping and unmap it if it's dropped to zero. */ 358 void mail_index_unmap(struct mail_index_map **map); 359 /* Clone a map. It still points to the original rec_map. */ 360 struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map); 361 /* Make sure the map has its own private rec_map, cloning it if necessary. */ 362 void mail_index_record_map_move_to_private(struct mail_index_map *map); 363 /* If map points to mmap()ed index, copy it to the memory. */ 364 void mail_index_map_move_to_memory(struct mail_index_map *map); 365 366 void mail_index_fchown(struct mail_index *index, int fd, const char *path); 367 368 bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name, 369 uint32_t *idx_r); 370 uint32_t 371 mail_index_map_register_ext(struct mail_index_map *map, 372 const char *name, uint32_t ext_offset, 373 const struct mail_index_ext_header *ext_hdr); 374 bool mail_index_map_get_ext_idx(struct mail_index_map *map, 375 uint32_t ext_id, uint32_t *idx_r); 376 const struct mail_index_ext * 377 mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id); 378 379 void mail_index_map_lookup_seq_range(struct mail_index_map *map, 380 uint32_t first_uid, uint32_t last_uid, 381 uint32_t *first_seq_r, 382 uint32_t *last_seq_r); 383 384 /* Returns 1 on success, 0 on non-critical errors we want to silently fix, 385 -1 if map isn't usable. The caller is responsible for logging the errors 386 if -1 is returned. */ 387 int mail_index_map_check_header(struct mail_index_map *map, 388 const char **error_r); 389 /* Returns 1 if header is usable, 0 or -1 if not. The caller should log an 390 error if -1 is returned, but not if 0 is returned. */ 391 bool mail_index_check_header_compat(struct mail_index *index, 392 const struct mail_index_header *hdr, 393 uoff_t file_size, const char **error_r); 394 int mail_index_map_parse_extensions(struct mail_index_map *map); 395 int mail_index_map_parse_keywords(struct mail_index_map *map); 396 397 void mail_index_map_init_extbufs(struct mail_index_map *map, 398 unsigned int initial_count); 399 int mail_index_map_ext_get_next(struct mail_index_map *map, 400 unsigned int *offset, 401 const struct mail_index_ext_header **ext_hdr_r, 402 const char **name_r); 403 int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr, 404 const struct mail_index_ext_header *ext_hdr, 405 const char *name, const char **error_r); 406 unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len); 407 408 void mail_index_fsck_locked(struct mail_index *index); 409 410 /* Log an error and set it as the index's current error that is available 411 with mail_index_get_error_message(). */ 412 void mail_index_set_error(struct mail_index *index, const char *fmt, ...) 413 ATTR_FORMAT(2, 3); 414 /* Same as mail_index_set_error(), but don't log the error. */ 415 void mail_index_set_error_nolog(struct mail_index *index, const char *str); 416 /* "%s failed with index file %s: %m" */ 417 void mail_index_set_syscall_error(struct mail_index *index, 418 const char *function); 419 /* "%s failed with file %s: %m" */ 420 void mail_index_file_set_syscall_error(struct mail_index *index, 421 const char *filepath, 422 const char *function); 423 424 #endif 425