1 #ifndef MAIL_TRANSACTION_LOG_H 2 #define MAIL_TRANSACTION_LOG_H 3 4 #include "mail-index.h" 5 6 #define MAIL_TRANSACTION_LOG_SUFFIX ".log" 7 8 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1 9 #define MAIL_TRANSACTION_LOG_MINOR_VERSION 3 10 /* Minimum allowed mail_transaction_log_header.hdr_size. If it's smaller, 11 assume the file is corrupted. */ 12 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24 13 14 /* Helper macro for other MAIL_TRANSACTION_LOG_VERSION_*() macros */ 15 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \ 16 ((major) << 8 | (minor)) 17 /* Returns TRUE if the transaction log version supports the given feature. 18 The wanted_feature is one of the MAIL_TRANSACTION_LOG_VERSION_FEATURE_* 19 macros without the macro prefix, e.g. just COMPAT_FLAGS. */ 20 #define MAIL_TRANSACTION_LOG_VERSION_HAVE(version, wanted_feature) \ 21 ((version) >= MAIL_TRANSACTION_LOG_VERSION_FEATURE_##wanted_feature) 22 /* Returns transaction log version from the given mail_transaction_log_header 23 which is compatible for the MAIL_TRANSACTION_LOG_VERSION_HAVE() macro. */ 24 #define MAIL_TRANSACTION_LOG_HDR_VERSION(hdr) \ 25 MAIL_TRANSACTION_LOG_VERSION_FULL((hdr)->major_version, (hdr)->minor_version) 26 27 /* Log feature: mail_transaction_log_header.compat_flags is filled. */ 28 #define MAIL_TRANSACTION_LOG_VERSION_FEATURE_COMPAT_FLAGS \ 29 MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2) 30 /* Log feature: Don't increase modseq when reading internal flag updates 31 (because they're not client-visible anyway). 32 See MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(). */ 33 #define MAIL_TRANSACTION_LOG_VERSION_FEATURE_HIDE_INTERNAL_MODSEQS \ 34 MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3) 35 36 struct mail_transaction_log_header { 37 /* Major version is increased only when you can't have backwards 38 compatibility. If the field doesn't match 39 MAIL_TRANSACTION_LOG_MAJOR_VERSION, don't even try to read it. */ 40 uint8_t major_version; 41 /* Minor version is increased when the file format changes in a 42 backwards compatible way. */ 43 uint8_t minor_version; 44 /* Size of the header. If it's larger than this struct, ignore any 45 unknown fields. If it's smaller, assume the rest of the fields 46 are 0. */ 47 uint16_t hdr_size; 48 49 /* Unique index file ID, which must match the main index's indexid. 50 See mail_index_header.indexid. This is overwritten to be 0 if the 51 log file is marked as corrupted. */ 52 uint32_t indexid; 53 /* Log file sequence number. Increased every time the log is rotated 54 and a new log is created. Using (file_seq, offset) uniquely 55 identifies a position in the transaction log. */ 56 uint32_t file_seq; 57 /* The previous log file's sequence and offset when the log was 58 rotated. The offset should be the same as the previous log file's 59 size. If there was no previous log file, or if the index is being 60 reset, these are 0. 61 62 These are mainly useful to optimize syncing when the start position 63 is (prev_file_seq, prev_file_offset). Then it's it's already known 64 that the syncing can be started from this log file wihtout having 65 to open the previous log file only to realize that there is nothing 66 to sync. (Which could have also lead to an error if the .log.2 was 67 already deleted.) */ 68 uint32_t prev_file_seq; 69 uint32_t prev_file_offset; 70 /* UNIX timestamp when this file was created. Used in determining when 71 to rotate the log file. */ 72 uint32_t create_stamp; 73 /* Modseq value at the beginning of this file. Some transaction records 74 increase the modseq value. (Only with log format v1.1+) */ 75 uint64_t initial_modseq; 76 77 /* Same as enum mail_index_header_compat_flags. Needs 78 MAIL_TRANSACTION_LOG_VERSION_FEATURE_COMPAT_FLAGS. */ 79 uint8_t compat_flags; 80 /* Unused fields to make the struct 64bit aligned. These can be used 81 to add more fields to the header. */ 82 uint8_t unused[3]; 83 uint32_t unused2; 84 }; 85 86 enum mail_transaction_type { 87 /* struct mail_transaction_expunge[] - Expunge the UIDs. 88 Must have MAIL_TRANSACTION_EXPUNGE_PROT ORed to this. Avoid using 89 this, use MAIL_TRANSACTION_EXPUNGE_GUID instead. */ 90 MAIL_TRANSACTION_EXPUNGE = 0x00000001, 91 /* struct mail_index_record[] - Save new mails with given flags. */ 92 MAIL_TRANSACTION_APPEND = 0x00000002, 93 /* struct mail_transaction_flag_update[] - Update message flags 94 (or just modseq). */ 95 MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, 96 /* struct mail_transaction_header_update[] - Update the index's base 97 header (struct mail_index_header). */ 98 MAIL_TRANSACTION_HEADER_UPDATE = 0x00000020, 99 /* struct mail_transaction_ext_intro - Start operations for the given 100 extension. This can be used to create a new extension or resize an 101 existing extension, but usually it is just used in front of the 102 other MAIL_TRANSACTION_EXT_* records to specify which extension 103 they're working with. */ 104 MAIL_TRANSACTION_EXT_INTRO = 0x00000040, 105 /* struct mail_transaction_ext_reset - Reset the last intro extension 106 by changing its reset_id and optionally zeroing out its old data. */ 107 MAIL_TRANSACTION_EXT_RESET = 0x00000080, 108 /* struct mail_transaction_ext_hdr_update[] - Update the last intro 109 extension's header. This might later become deprecated in favor of 110 supporting only MAIL_TRANSACTION_EXT_HDR_UPDATE32, but for now 111 it's still used for <64kB headers. */ 112 MAIL_TRANSACTION_EXT_HDR_UPDATE = 0x00000100, 113 /* struct mail_transaction_ext_rec_update[] - Update the last intro 114 extension records for the given UIDs with given content. */ 115 MAIL_TRANSACTION_EXT_REC_UPDATE = 0x00000200, 116 /* struct mail_transaction_keyword_update - Add/remove the specified 117 keyword to messages. */ 118 MAIL_TRANSACTION_KEYWORD_UPDATE = 0x00000400, 119 /* struct mail_transaction_keyword_reset[] - Clear out all keywords 120 in specified messages. */ 121 MAIL_TRANSACTION_KEYWORD_RESET = 0x00000800, 122 /* struct mail_transaction_ext_atomic_inc[] - Atomically increase or 123 decrease the last intro extension record. The record must be 1, 2, 124 4 or 8 bytes. This can be used e.g. for refcount extensions. */ 125 MAIL_TRANSACTION_EXT_ATOMIC_INC = 0x00001000, 126 /* struct mail_transaction_expunge_guid[] - Expunge given UID, but 127 first verify that it matches the given GUID. Must have 128 MAIL_TRANSACTION_EXPUNGE_PROT ORed to this. */ 129 MAIL_TRANSACTION_EXPUNGE_GUID = 0x00002000, 130 MAIL_TRANSACTION_MODSEQ_UPDATE = 0x00008000, 131 /* struct mail_transaction_ext_hdr_update32[] - Update the last intro 132 extension's header. Used for >=64kB headers. See also 133 MAIL_TRANSACTION_EXT_HDR_UPDATE. This was added in Dovecot v2.0. */ 134 MAIL_TRANSACTION_EXT_HDR_UPDATE32 = 0x00010000, 135 /* Index was marked as deleted using mail_index_set_deleted(). 136 There is no record content for this. */ 137 MAIL_TRANSACTION_INDEX_DELETED = 0x00020000, 138 /* Index was marked as undeleted using mail_index_set_undeleted(). 139 There is no record content for this. */ 140 MAIL_TRANSACTION_INDEX_UNDELETED = 0x00040000, 141 /* struct mail_transaction_boundary - Specifies a size of the following 142 records that must be treated as a single transaction. This works 143 so that the transaction log reading code stops if it finds that 144 there is a transaction whose size points outside the currently 145 existing file. An unfinished transaction is truncated away after the 146 next write to the log. FIXME: it would be better to rotate the 147 log instead of truncating it. */ 148 MAIL_TRANSACTION_BOUNDARY = 0x00080000, 149 /* Mailbox attribute update. This is a bit complicated format: 150 - [+-][p-s]<name><NUL> 151 - "+" means attribute is set, "-" means unset 152 - "p" means private attribute, "s" means shared 153 - <name> is the attribute name 154 - This can repeat multiple times 155 - <NUL> 156 - 0..3 bytes padding for 32bit alignment 157 - For each attribute update an array of uint32_t integers: 158 - Update timestamp 159 - For each "+" only: Length of the attribute value. 160 */ 161 MAIL_TRANSACTION_ATTRIBUTE_UPDATE = 0x00100000, 162 163 /* Mask to get the attribute type only (excluding flags). */ 164 MAIL_TRANSACTION_TYPE_MASK = 0x0fffffff, 165 166 #define MAIL_TRANSACTION_EXT_MASK \ 167 (MAIL_TRANSACTION_EXT_INTRO | MAIL_TRANSACTION_EXT_RESET | \ 168 MAIL_TRANSACTION_EXT_HDR_UPDATE | MAIL_TRANSACTION_EXT_HDR_UPDATE32 | \ 169 MAIL_TRANSACTION_EXT_REC_UPDATE | MAIL_TRANSACTION_EXT_ATOMIC_INC) 170 171 /* Since we'll expunge mails based on data read from transaction log, 172 try to avoid the possibility of corrupted transaction log expunging 173 messages. This value is ORed to the actual MAIL_TRANSACTION_EXPUNGE* 174 flag. If it's not present, assume corrupted log. */ 175 MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90, 176 177 /* External transactions have a bit different meanings depending on the 178 transaction type. Generally they mean to indicate changes that have 179 already occurred, instead of changes that are only being requested 180 to happen on next sync. For example expunges are first requested 181 to be done with internal transactions, and then there's a separate 182 external transaction to indicate that they were actually done. */ 183 MAIL_TRANSACTION_EXTERNAL = 0x10000000, 184 /* This change syncs the state with another mailbox (dsync), 185 i.e. the change isn't something that a user requested locally. */ 186 MAIL_TRANSACTION_SYNC = 0x20000000 187 }; 188 189 struct mail_transaction_header { 190 /* Size of this header and the following records. This size can be 191 used to calculate how many records there are. The size is written 192 via mail_index_uint32_to_offset(). */ 193 uint32_t size; 194 uint32_t type; /* enum mail_transaction_type */ 195 /* Header is followed by the type-specific records. */ 196 }; 197 198 /* See MAIL_TRANSACTION_MODSEQ_UPDATE. */ 199 struct mail_transaction_modseq_update { 200 uint32_t uid; 201 /* don't use uint64_t here. it adds extra 32 bits of padding and also 202 causes problems with CPUs that require alignment */ 203 uint32_t modseq_low32; 204 uint32_t modseq_high32; 205 }; 206 207 /* See MAIL_TRANSACTION_EXPUNGE. */ 208 struct mail_transaction_expunge { 209 /* Expunge all mails between uid1..uid2. */ 210 uint32_t uid1, uid2; 211 }; 212 /* See MAIL_TRANSACTION_EXPUNGE_GUID. */ 213 struct mail_transaction_expunge_guid { 214 /* Expunge uid, but only if it matches guid_128. */ 215 uint32_t uid; 216 /* GUID of the mail. If it's not 128 bit GUID, first pass it through 217 mail_generate_guid_128_hash() to get 128 bit SHA1 of it. */ 218 guid_128_t guid_128; 219 }; 220 221 /* See MAIL_TRANSACTION_FLAG_UPDATE. */ 222 struct mail_transaction_flag_update { 223 /* Change the flags for all mails between uid1..uid2. */ 224 uint32_t uid1, uid2; 225 /* Add these flags to the mails. */ 226 uint8_t add_flags; 227 /* Remove these flags to the mails. To replace all existing flags, 228 just set this to 0xff and specify the wanted flags in add_flags. */ 229 uint8_t remove_flags; 230 /* If non-0, MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ was used to force 231 increasing modseq update to the mails even though no flags were 232 actually changed. This differs from MAIL_TRANSACTION_MODSEQ_UPDATE 233 in that the modseq is just wanted to be increased, doesn't matter 234 to which value specifically. */ 235 uint8_t modseq_inc_flag; 236 /* Unused padding */ 237 uint8_t padding; 238 }; 239 240 /* See MAIL_TRANSACTION_KEYWORD_UPDATE. */ 241 struct mail_transaction_keyword_update { 242 /* enum modify_type : MODIFY_ADD / MODIFY_REMOVE */ 243 uint8_t modify_type; 244 uint8_t padding; 245 /* Size of name[] */ 246 uint16_t name_size; 247 /* unsigned char name[name_size]; */ 248 /* Update keywords for the given UIDs. The array's size is calculated 249 from mail_transaction_header.size. */ 250 /* array of { uint32_t uid1, uid2; } */ 251 }; 252 253 /* See MAIL_TRANSACTION_KEYWORD_RESET. */ 254 struct mail_transaction_keyword_reset { 255 /* Clear out all keywords for uid1..uid2. */ 256 uint32_t uid1, uid2; 257 }; 258 259 /* See MAIL_TRANSACTION_HEADER_UPDATE. */ 260 struct mail_transaction_header_update { 261 /* Update start offset. */ 262 uint16_t offset; 263 /* Size of the following data[] to update. */ 264 uint16_t size; 265 /* unsigned char data[size]; */ 266 /* 0..3 bytes of padding to get to 32bit alignment. */ 267 /* unsigned char padding[]; */ 268 }; 269 270 enum { 271 /* Don't shrink hdr_size, record_size or record_align but grow them 272 if necessary. */ 273 MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK = 0x01 274 }; 275 276 /* See MAIL_TRANSACTION_EXT_INTRO. Also see struct mail_index_ext_header for 277 more explanations of these fields. */ 278 struct mail_transaction_ext_intro { 279 /* If extension is already known to exist in the index file, 280 set ext_id, but use empty name. If this is a new extension, set 281 name, but use ext_id=(uint32_t)-1. */ 282 uint32_t ext_id; 283 uint32_t reset_id; 284 /* Size of the extension header. When growing the header size, it's 285 initially filled with zeros. The header can be written to with 286 ext-hdr-update records. */ 287 uint32_t hdr_size; 288 uint16_t record_size; 289 uint16_t record_align; 290 uint16_t flags; 291 uint16_t name_size; 292 /* unsigned char name[]; */ 293 }; 294 295 /* See MAIL_TRANSACTION_EXT_RESET. */ 296 struct mail_transaction_ext_reset { 297 /* New value for extension's reset_id */ 298 uint32_t new_reset_id; 299 /* Non-0 if the old extension header and record data should be 300 preserved. Normally all of it is zeroed out. */ 301 uint8_t preserve_data; 302 uint8_t unused_padding[3]; 303 }; 304 305 /* See MAIL_TRANSACTION_EXT_HDR_UPDATE. */ 306 struct mail_transaction_ext_hdr_update { 307 /* Update start offset. */ 308 uint16_t offset; 309 /* Size of the following data[] to update. */ 310 uint16_t size; 311 /* unsigned char data[size]; */ 312 /* 0..3 bytes of padding to get to 32bit alignment. */ 313 /* unsigned char padding[]; */ 314 }; 315 /* See MAIL_TRANSACTION_EXT_HDR_UPDATE32. */ 316 struct mail_transaction_ext_hdr_update32 { 317 /* Update start offset. */ 318 uint32_t offset; 319 /* Size of the following data[] to update. */ 320 uint32_t size; 321 /* unsigned char data[size]; */ 322 /* 0..3 bytes of padding to get to 32bit alignment. */ 323 /* unsigned char padding[]; */ 324 }; 325 326 /* See MAIL_TRANSACTION_EXT_REC_UPDATE. */ 327 struct mail_transaction_ext_rec_update { 328 uint32_t uid; 329 /* unsigned char data[mail_transaction_ext_intro.record_size]; */ 330 /* 0..3 bytes of padding to get to 32bit alignment. */ 331 /* unsigned char padding[]; */ 332 }; 333 334 /* See MAIL_TRANSACTION_EXT_ATOMIC_INC. */ 335 struct mail_transaction_ext_atomic_inc { 336 uint32_t uid; 337 /* Add this value to the extension record data. Can be negative. */ 338 int32_t diff; 339 }; 340 341 /* See MAIL_TRANSACTION_BOUNDARY. */ 342 struct mail_transaction_boundary { 343 /* Size of the whole transaction, including this record and header. */ 344 uint32_t size; 345 }; 346 347 struct mail_transaction_log_append_ctx { 348 struct mail_transaction_log *log; 349 /* All the changes that will be written to the transaction log. */ 350 buffer_t *output; 351 352 /* Transaction flags as given to mail_transaction_log_append_begin(). */ 353 enum mail_transaction_type trans_flags; 354 355 /* Tracking the current highest_modseq after the changes. This will 356 be used to update mail_transaction_log_file.sync_highest_modseq. */ 357 uint64_t new_highest_modseq; 358 /* Number of transaction records added so far. */ 359 unsigned int transaction_count; 360 361 /* Copied from mail_index_transaction.sync_transaction */ 362 bool index_sync_transaction:1; 363 /* Copied from mail_index_transaction.tail_offset_changed */ 364 bool tail_offset_changed:1; 365 /* TRUE if the mail_transaction_log_file has been synced up to the 366 current write offset, and we're writing a syncing transaction 367 (index_sync_transaction=TRUE). This means that the just written 368 transaction can be assumed to be synced already. */ 369 bool sync_includes_this:1; 370 /* fdatasync() after writing the transaction. */ 371 bool want_fsync:1; 372 }; 373 374 #define LOG_IS_BEFORE(seq1, offset1, seq2, offset2) \ 375 (((offset1) < (offset2) && (seq1) == (seq2)) || (seq1) < (seq2)) 376 377 struct mail_transaction_log * 378 mail_transaction_log_alloc(struct mail_index *index); 379 void mail_transaction_log_free(struct mail_transaction_log **log); 380 381 /* Open the transaction log. Returns 1 if ok, 0 if file doesn't exist or it's 382 is corrupted, -1 if there was some I/O error. */ 383 int mail_transaction_log_open(struct mail_transaction_log *log); 384 /* Create, or recreate, the transaction log. Returns 0 if ok, -1 if error. */ 385 int mail_transaction_log_create(struct mail_transaction_log *log, bool reset); 386 /* Close all the open transactions log files. */ 387 void mail_transaction_log_close(struct mail_transaction_log *log); 388 389 /* Notify of indexid change */ 390 void mail_transaction_log_indexid_changed(struct mail_transaction_log *log); 391 392 /* Returns the file seq/offset where the mailbox is currently synced at. 393 Since the log is rotated only when mailbox is fully synced, the sequence 394 points always to the latest file. This function doesn't actually find the 395 latest sync position, so you'll need to use eg. log_view_set() before 396 calling this. */ 397 void mail_transaction_log_get_mailbox_sync_pos(struct mail_transaction_log *log, 398 uint32_t *file_seq_r, 399 uoff_t *file_offset_r); 400 /* Set the current mailbox sync position. file_seq must always be the latest 401 log file's sequence. The offset written automatically to the log when 402 other transactions are being written. */ 403 void mail_transaction_log_set_mailbox_sync_pos(struct mail_transaction_log *log, 404 uint32_t file_seq, 405 uoff_t file_offset); 406 407 struct mail_transaction_log_view * 408 mail_transaction_log_view_open(struct mail_transaction_log *log); 409 void mail_transaction_log_view_close(struct mail_transaction_log_view **view); 410 411 /* Set view boundaries. Returns 1 if ok, 0 if files are lost, corrupted or the 412 offsets are broken, -1 if I/O error. reset_r=TRUE if the whole index should 413 be reset before applying any changes. */ 414 int mail_transaction_log_view_set(struct mail_transaction_log_view *view, 415 uint32_t min_file_seq, uoff_t min_file_offset, 416 uint32_t max_file_seq, uoff_t max_file_offset, 417 bool *reset_r, const char **reason_r); 418 /* Scan through all of the log files that we can find. 419 Returns -1 if error, 0 if ok. */ 420 int mail_transaction_log_view_set_all(struct mail_transaction_log_view *view); 421 /* Clear the view. If oldest_file_seq > 0, keep it and newer log files 422 referenced so we don't get desynced. */ 423 void mail_transaction_log_view_clear(struct mail_transaction_log_view *view, 424 uint32_t oldest_file_seq); 425 426 /* Read next transaction record from current position. The position is updated. 427 Returns -1 if error, 0 if we're at end of the view, 1 if ok. */ 428 int mail_transaction_log_view_next(struct mail_transaction_log_view *view, 429 const struct mail_transaction_header **hdr_r, 430 const void **data_r); 431 /* Mark the current view's position to the record returned previously with 432 _log_view_next(). */ 433 void mail_transaction_log_view_mark(struct mail_transaction_log_view *view); 434 /* Seek to previously marked position. */ 435 void mail_transaction_log_view_rewind(struct mail_transaction_log_view *view); 436 437 /* Returns the position of the record returned previously with 438 mail_transaction_log_view_next() */ 439 void 440 mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view, 441 uint32_t *file_seq_r, 442 uoff_t *file_offset_r); 443 /* Return the modseq of the change returned previously with _view_next(). */ 444 uint64_t 445 mail_transaction_log_view_get_prev_modseq(struct mail_transaction_log_view *view); 446 /* Returns TRUE if we're at the end of the view window. */ 447 bool mail_transaction_log_view_is_last(struct mail_transaction_log_view *view); 448 449 /* Marks the log file in current position to be corrupted. */ 450 void 451 mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view, 452 const char *fmt, ...) 453 ATTR_FORMAT(2, 3); 454 bool 455 mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view); 456 457 int mail_transaction_log_append_begin(struct mail_index *index, 458 enum mail_transaction_type flags, 459 struct mail_transaction_log_append_ctx **ctx_r); 460 void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx, 461 enum mail_transaction_type type, 462 const void *data, size_t size); 463 int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **ctx); 464 465 /* Lock transaction log for index synchronization. This is used as the main 466 exclusive lock for index changes. The index/log can still be read since they 467 don't use locking, but the log can't be written to while it's locked. 468 Returns 0 on success, -1 if locking failed for any reason. 469 470 After successfully locking the transaction log, the log file is also fully 471 mapped into memory and its sync_offset updated. The locked file's sequence 472 and sync_offset are returned. */ 473 int mail_transaction_log_sync_lock(struct mail_transaction_log *log, 474 const char *lock_reason, 475 uint32_t *file_seq_r, uoff_t *file_offset_r); 476 void mail_transaction_log_sync_unlock(struct mail_transaction_log *log, 477 const char *lock_reason); 478 /* Returns the current head. Works only when log is locked. */ 479 void mail_transaction_log_get_head(struct mail_transaction_log *log, 480 uint32_t *file_seq_r, uoff_t *file_offset_r); 481 /* Returns the current tail from which all files are open to head. */ 482 void mail_transaction_log_get_tail(struct mail_transaction_log *log, 483 uint32_t *file_seq_r); 484 /* Returns TRUE if given seq/offset is current head log's rotate point. */ 485 bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log, 486 uint32_t file_seq, uoff_t file_offset); 487 488 /* Move currently opened log head file to memory (called by 489 mail_index_move_to_memory()) */ 490 int mail_transaction_log_move_to_memory(struct mail_transaction_log *log); 491 /* Unlink transaction log files */ 492 int mail_transaction_log_unlink(struct mail_transaction_log *log); 493 494 #endif 495