1 /* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "istream.h"
5 #include "hex-binary.h"
6 #include "mail-index-private.h"
7 #include "mail-transaction-log-private.h"
8 #include "doveadm-dump.h"
9
10 #include <stdio.h>
11
12 static struct mail_transaction_ext_intro prev_intro;
13
dump_hdr(struct istream * input,uint64_t * modseq_r,unsigned int * version_r)14 static void dump_hdr(struct istream *input, uint64_t *modseq_r,
15 unsigned int *version_r)
16 {
17 struct mail_transaction_log_header hdr;
18 const unsigned char *data;
19 size_t size;
20 int ret;
21
22 ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr));
23 if (ret < 0 && input->stream_errno != 0)
24 i_fatal("read() failed: %s", i_stream_get_error(input));
25 if (ret <= 0) {
26 i_fatal("file hdr read() %zu != %zu",
27 size, sizeof(hdr));
28 }
29 memcpy(&hdr, data, sizeof(hdr));
30 if (hdr.hdr_size < sizeof(hdr)) {
31 memset(PTR_OFFSET(&hdr, hdr.hdr_size), 0,
32 sizeof(hdr) - hdr.hdr_size);
33 }
34 i_stream_skip(input, hdr.hdr_size);
35
36 printf("version = %u.%u\n", hdr.major_version, hdr.minor_version);
37 printf("hdr size = %u\n", hdr.hdr_size);
38 printf("index id = %u\n", hdr.indexid);
39 printf("file seq = %u\n", hdr.file_seq);
40 printf("prev file = %u/%u\n", hdr.prev_file_seq, hdr.prev_file_offset);
41 printf("create stamp = %u\n", hdr.create_stamp);
42 printf("initial modseq = %"PRIu64"\n", hdr.initial_modseq);
43 printf("compat flags = %x\n", hdr.compat_flags);
44 *modseq_r = hdr.initial_modseq;
45 *version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
46 }
47
log_record_type(unsigned int type)48 static const char *log_record_type(unsigned int type)
49 {
50 const char *name;
51
52 switch (type & MAIL_TRANSACTION_TYPE_MASK) {
53 case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT:
54 name = "expunge";
55 break;
56 case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT:
57 name = "expunge-guid";
58 break;
59 case MAIL_TRANSACTION_APPEND:
60 name = "append";
61 break;
62 case MAIL_TRANSACTION_FLAG_UPDATE:
63 name = "flag-update";
64 break;
65 case MAIL_TRANSACTION_HEADER_UPDATE:
66 name = "header-update";
67 break;
68 case MAIL_TRANSACTION_EXT_INTRO:
69 name = "ext-intro";
70 break;
71 case MAIL_TRANSACTION_EXT_RESET:
72 name = "ext-reset";
73 break;
74 case MAIL_TRANSACTION_EXT_HDR_UPDATE:
75 name = "ext-hdr";
76 break;
77 case MAIL_TRANSACTION_EXT_HDR_UPDATE32:
78 name = "ext-hdr32";
79 break;
80 case MAIL_TRANSACTION_EXT_REC_UPDATE:
81 name = "ext-rec";
82 break;
83 case MAIL_TRANSACTION_KEYWORD_UPDATE:
84 name = "keyword-update";
85 break;
86 case MAIL_TRANSACTION_KEYWORD_RESET:
87 name = "keyword-reset";
88 break;
89 case MAIL_TRANSACTION_EXT_ATOMIC_INC:
90 name = "ext-atomic-inc";
91 break;
92 case MAIL_TRANSACTION_MODSEQ_UPDATE:
93 name = "modseq-update";
94 break;
95 case MAIL_TRANSACTION_INDEX_DELETED:
96 name = "index-deleted";
97 break;
98 case MAIL_TRANSACTION_INDEX_UNDELETED:
99 name = "index-undeleted";
100 break;
101 case MAIL_TRANSACTION_BOUNDARY:
102 name = "boundary";
103 break;
104 case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
105 name = "attribute-update";
106 break;
107 default:
108 name = t_strdup_printf("unknown: %x", type);
109 break;
110 }
111
112 if ((type & MAIL_TRANSACTION_EXTERNAL) != 0)
113 name = t_strconcat(name, " (ext)", NULL);
114 if ((type & MAIL_TRANSACTION_SYNC) != 0)
115 name = t_strconcat(name, " (sync)", NULL);
116 return name;
117 }
118
print_data(const void * data,size_t size)119 static void print_data(const void *data, size_t size)
120 {
121 size_t i;
122
123 for (i = 0; i < size; i++)
124 printf("%02x", ((const unsigned char *)data)[i]);
125 if (size == 4) {
126 const uint32_t *n = (const uint32_t *)data;
127
128 printf(" (dec=%u)", *n);
129 }
130 }
131
print_try_uint(const void * data,size_t size)132 static void print_try_uint(const void *data, size_t size)
133 {
134 size_t i;
135
136 switch (size) {
137 case 1: {
138 const uint8_t *n = data;
139 printf("%u", *n);
140 break;
141 }
142 case 2: {
143 const uint16_t *n = data;
144 uint32_t n16;
145
146 memcpy(&n16, n, sizeof(n16));
147 printf("%u", n16);
148 break;
149 }
150 case 4: {
151 const uint32_t *n = data;
152 uint32_t n32;
153
154 memcpy(&n32, n, sizeof(n32));
155 printf("%u", n32);
156 break;
157 }
158 case 8: {
159 const uint64_t *n = data;
160 uint64_t n64;
161
162 memcpy(&n64, n, sizeof(n64));
163 printf("%"PRIu64, n64);
164 break;
165 }
166 default:
167 for (i = 0; i < size; i++)
168 printf("%02x", ((const unsigned char *)data)[i]);
169 }
170 }
171
172 #define HDRF(field) { \
173 #field, offsetof(struct mail_index_header, field), \
174 sizeof(((struct mail_index_header *)0)->field) }
175
176 static struct {
177 const char *name;
178 unsigned int offset, size;
179 } header_fields[] = {
180 HDRF(minor_version),
181 HDRF(base_header_size),
182 HDRF(header_size),
183 HDRF(record_size),
184 HDRF(compat_flags),
185 HDRF(indexid),
186 HDRF(flags),
187 HDRF(uid_validity),
188 HDRF(next_uid),
189 HDRF(messages_count),
190 HDRF(unused_old_recent_messages_count),
191 HDRF(seen_messages_count),
192 HDRF(deleted_messages_count),
193 HDRF(first_recent_uid),
194 HDRF(first_unseen_uid_lowwater),
195 HDRF(first_deleted_uid_lowwater),
196 HDRF(log_file_seq),
197 HDRF(log_file_tail_offset),
198 HDRF(log_file_head_offset),
199 HDRF(day_stamp)
200 };
201
log_header_update(const struct mail_transaction_header_update * u,size_t data_size)202 static void log_header_update(const struct mail_transaction_header_update *u,
203 size_t data_size)
204 {
205 const void *data = u + 1;
206 unsigned int offset = u->offset, size = u->size;
207 unsigned int i;
208
209 if (sizeof(*u) + size > data_size) {
210 printf(" - offset = %u, size = %u (too large)\n", offset, size);
211 return;
212 }
213
214 while (size > 0) {
215 /* don't bother trying to handle header updates that include
216 unknown/unexpected fields offsets/sizes */
217 for (i = 0; i < N_ELEMENTS(header_fields); i++) {
218 if (header_fields[i].offset == offset &&
219 header_fields[i].size <= size)
220 break;
221 }
222
223 if (i == N_ELEMENTS(header_fields)) {
224 printf(" - offset = %u, size = %u: ", offset, size);
225 print_data(data, size);
226 printf("\n");
227 break;
228 }
229
230 printf(" - %s = ", header_fields[i].name);
231 print_try_uint(data, header_fields[i].size);
232 printf("\n");
233
234 data = CONST_PTR_OFFSET(data, header_fields[i].size);
235 offset += header_fields[i].size;
236 size -= header_fields[i].size;
237 }
238 }
239
log_record_print(const struct mail_transaction_header * hdr,const void * data,size_t data_size,uint64_t * modseq)240 static void log_record_print(const struct mail_transaction_header *hdr,
241 const void *data, size_t data_size,
242 uint64_t *modseq)
243 {
244 unsigned int size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr);
245
246 switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
247 case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
248 const struct mail_transaction_expunge *exp = data;
249
250 printf(" - uids=");
251 for (; size > 0; size -= sizeof(*exp), exp++) {
252 printf("%u-%u,", exp->uid1, exp->uid2);
253 }
254 printf("\n");
255 break;
256 }
257 case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
258 const struct mail_transaction_expunge_guid *exp = data;
259
260 for (; size > 0; size -= sizeof(*exp), exp++) {
261 printf(" - uid=%u (guid ", exp->uid);
262 print_data(exp->guid_128, sizeof(exp->guid_128));
263 printf(")\n");
264 }
265 break;
266 }
267 case MAIL_TRANSACTION_APPEND: {
268 const struct mail_index_record *rec = data;
269
270 printf(" - uids=");
271 for (; size > 0; size -= sizeof(*rec), rec++) {
272 printf("%u", rec->uid);
273 if (rec->flags != 0)
274 printf(" (flags=%x)", rec->flags);
275 printf(",");
276 }
277 printf("\n");
278 break;
279 }
280 case MAIL_TRANSACTION_FLAG_UPDATE: {
281 const struct mail_transaction_flag_update *u = data;
282
283 for (; size > 0; size -= sizeof(*u), u++) {
284 printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n",
285 u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag);
286 }
287 break;
288 }
289 case MAIL_TRANSACTION_HEADER_UPDATE: {
290 const struct mail_transaction_header_update *u = data;
291
292 log_header_update(u, data_size);
293 break;
294 }
295 case MAIL_TRANSACTION_EXT_INTRO: {
296 const struct mail_transaction_ext_intro *intro = data;
297
298 prev_intro = *intro;
299 printf(" - ext_id = %u\n", intro->ext_id);
300 printf(" - reset_id = %u\n", intro->reset_id);
301 printf(" - hdr_size = %u\n", intro->hdr_size);
302 printf(" - record_size = %u\n", intro->record_size);
303 printf(" - record_align = %u\n", intro->record_align);
304 printf(" - flags = %u\n", intro->flags);
305 printf(" - name_size = %u\n", intro->name_size);
306 if (intro->name_size > 0) {
307 const char *name = (const char *)(intro+1);
308
309 printf(" - name = '%.*s'\n", intro->name_size, name);
310 if (*modseq == 0 && intro->name_size == 6 &&
311 memcmp(name, "modseq", 6) == 0)
312 *modseq = 1;
313 }
314 break;
315 }
316 case MAIL_TRANSACTION_EXT_RESET: {
317 const struct mail_transaction_ext_reset *reset = data;
318
319 printf(" - new_reset_id = %u\n", reset->new_reset_id);
320 printf(" - preserve_data = %u\n", reset->preserve_data);
321 break;
322 }
323 case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
324 const struct mail_transaction_ext_hdr_update *u = data;
325
326 printf(" - offset = %u, size = %u", u->offset, u->size);
327 if (sizeof(*u) + u->size <= data_size) {
328 printf(": ");
329 print_data(u + 1, u->size);
330 } else {
331 printf(" (too large)");
332 }
333 printf("\n");
334 break;
335 }
336 case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
337 const struct mail_transaction_ext_hdr_update32 *u = data;
338
339 printf(" - offset = %u, size = %u", u->offset, u->size);
340 if (sizeof(*u) + u->size <= data_size) {
341 printf(": ");
342 print_data(u + 1, u->size);
343 } else {
344 printf(" (too large)");
345 }
346 printf("\n");
347 break;
348 }
349 case MAIL_TRANSACTION_EXT_REC_UPDATE: {
350 const struct mail_transaction_ext_rec_update *rec = data, *end;
351 size_t record_size;
352
353 end = CONST_PTR_OFFSET(data, size);
354 record_size = (sizeof(*rec) + prev_intro.record_size + 3) & ~3U;
355 while (rec < end) {
356 printf(" - uid=%u: ", rec->uid);
357 size_t bytes_left = (const char *)end - (const char *)(rec + 1);
358 if (prev_intro.record_size <= bytes_left)
359 print_data(rec + 1, prev_intro.record_size);
360 else
361 printf("(record_size too large)");
362 printf("\n");
363 rec = CONST_PTR_OFFSET(rec, record_size);
364 }
365 break;
366 }
367 case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
368 const struct mail_transaction_ext_atomic_inc *rec = data, *end;
369
370 end = CONST_PTR_OFFSET(data, size);
371 for (; rec < end; rec++) {
372 printf(" - uid=%u: ", rec->uid);
373 if (rec->diff > 0)
374 printf("+%d\n", rec->diff);
375 else
376 printf("%d\n", rec->diff);
377 }
378 break;
379 }
380 case MAIL_TRANSACTION_KEYWORD_UPDATE: {
381 const struct mail_transaction_keyword_update *u = data;
382 const uint32_t *uid;
383 unsigned int uid_offset;
384
385 printf(" - modify=%d, name=%.*s, uids=",
386 u->modify_type, u->name_size, (const char *)(u+1));
387
388 uid_offset = sizeof(*u) + u->name_size +
389 ((u->name_size % 4) == 0 ? 0 : 4 - (u->name_size%4));
390 uid = (const uint32_t *)((const char *)u + uid_offset);
391 size -= uid_offset;
392
393 for (; size > 0; size -= sizeof(*uid)*2, uid += 2) {
394 printf("%u-%u,", uid[0], uid[1]);
395 }
396 printf("\n");
397 break;
398 }
399 case MAIL_TRANSACTION_KEYWORD_RESET: {
400 const struct mail_transaction_keyword_reset *u = data;
401
402 printf(" - uids=");
403 for (; size > 0; size -= sizeof(*u), u++) {
404 printf("%u-%u, ", u->uid1, u->uid2);
405 }
406 printf("\n");
407 break;
408 }
409 case MAIL_TRANSACTION_MODSEQ_UPDATE: {
410 const struct mail_transaction_modseq_update *rec, *end;
411
412 end = CONST_PTR_OFFSET(data, size);
413 for (rec = data; rec < end; rec++) {
414 printf(" - uid=%u modseq=%"PRIu64"\n", rec->uid,
415 ((uint64_t)rec->modseq_high32 << 32) |
416 rec->modseq_low32);
417 }
418 break;
419 }
420 case MAIL_TRANSACTION_INDEX_DELETED:
421 case MAIL_TRANSACTION_INDEX_UNDELETED:
422 break;
423 case MAIL_TRANSACTION_BOUNDARY: {
424 const struct mail_transaction_boundary *rec = data;
425
426 printf(" - size=%u\n", rec->size);
427 break;
428 }
429 case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: {
430 const char *keys = data;
431 const uint32_t *extra;
432 unsigned int i, extra_pos, extra_count = 0;
433
434 for (i = 0; i < size && keys[i] != '\0'; ) {
435 if (keys[i] == '+')
436 extra_count++;
437 extra_count++;
438 i += strlen(keys+i) + 1;
439 }
440 if (i % sizeof(uint32_t) != 0)
441 i += sizeof(uint32_t) - i%sizeof(uint32_t);
442 extra = (const void *)(keys+i);
443
444 if ((size-i) != extra_count*sizeof(uint32_t)) {
445 printf(" - broken entry\n");
446 break;
447 }
448
449 extra_pos = 0;
450 for (i = 0; i < size && keys[i] != '\0'; ) {
451 printf(" - %s: %s/%s : timestamp=%s",
452 keys[i] == '+' ? "add" : keys[i] == '-' ? "remove" : "?",
453 keys[i+1] == 'p' ? "private" :
454 keys[i+1] == 's' ? "shared" : "?error?",
455 keys+i+2, unixdate2str(extra[extra_pos++]));
456 if (keys[i] == '+')
457 printf(" value_len=%u", extra[extra_pos++]);
458 printf("\n");
459 i += strlen(keys+i) + 1;
460 }
461
462 break;
463 }
464 default:
465 break;
466 }
467 }
468
dump_record(struct istream * input,uint64_t * modseq,unsigned int version)469 static int dump_record(struct istream *input, uint64_t *modseq,
470 unsigned int version)
471 {
472 struct mail_transaction_header hdr;
473 unsigned int hdr_size;
474 const unsigned char *data;
475 size_t size;
476 int ret;
477
478 ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr));
479 if (ret < 0 && input->stream_errno != 0)
480 i_fatal("read() failed: %s", i_stream_get_error(input));
481 if (ret <= 0) {
482 if (size == 0)
483 return 0;
484 i_fatal("rec hdr read() %zu != %zu",
485 size, sizeof(hdr));
486 }
487 memcpy(&hdr, data, sizeof(hdr));
488
489 hdr_size = mail_index_offset_to_uint32(hdr.size);
490 if (hdr_size < sizeof(hdr)) {
491 printf("record: offset=%"PRIuUOFF_T", "
492 "type=%s, size=broken (%x)\n",
493 input->v_offset, log_record_type(hdr.type), hdr.size);
494 return 0;
495 }
496
497 printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u",
498 input->v_offset, log_record_type(hdr.type), hdr_size);
499
500 i_stream_skip(input, sizeof(hdr));
501 size_t data_size = hdr_size - sizeof(hdr);
502 ret = i_stream_read_bytes(input, &data, &size, data_size);
503 if (ret < 0 && input->stream_errno != 0)
504 i_fatal("read() failed: %s", i_stream_get_error(input));
505 if (ret <= 0) {
506 i_fatal("rec data read() %zu != %zu",
507 size, data_size);
508 }
509
510 uint64_t prev_modseq = *modseq;
511 mail_transaction_update_modseq(&hdr, data, modseq, version);
512 if (*modseq > prev_modseq)
513 printf(", modseq=%"PRIu64, *modseq);
514 printf("\n");
515
516 log_record_print(&hdr, data, data_size, modseq);
517 i_stream_skip(input, data_size);
518 return 1;
519 }
520
cmd_dump_log(const char * path,const char * const * args ATTR_UNUSED)521 static void cmd_dump_log(const char *path, const char *const *args ATTR_UNUSED)
522 {
523 struct istream *input;
524 uint64_t modseq;
525 unsigned int version;
526 int ret;
527
528 input = i_stream_create_file(path, SIZE_MAX);
529 dump_hdr(input, &modseq, &version);
530 do {
531 T_BEGIN {
532 ret = dump_record(input, &modseq, version);
533 } T_END;
534 } while (ret > 0);
535 i_stream_unref(&input);
536 }
537
test_dump_log(const char * path)538 static bool test_dump_log(const char *path)
539 {
540 struct mail_transaction_log_header hdr;
541 const char *p;
542 bool ret = FALSE;
543 int fd;
544
545 p = strrchr(path, '/');
546 if (p == NULL)
547 p = path;
548 p = strstr(p, ".log");
549 if (p == NULL || !(p[4] == '\0' || p[4] == '.'))
550 return FALSE;
551
552 fd = open(path, O_RDONLY);
553 if (fd == -1)
554 return FALSE;
555
556 if (read(fd, &hdr, sizeof(hdr)) >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE &&
557 hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
558 hdr.hdr_size >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE)
559 ret = TRUE;
560 i_close_fd(&fd);
561 return ret;
562 }
563
564 struct doveadm_cmd_dump doveadm_cmd_dump_log = {
565 "log",
566 test_dump_log,
567 cmd_dump_log
568 };
569