1 /* mailbox.c -- Mailbox manipulation routines
2 *
3 * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
15 * distribution.
16 *
17 * 3. The name "Carnegie Mellon University" must not be used to
18 * endorse or promote products derived from this software without
19 * prior written permission. For permission or any legal
20 * details, please contact
21 * Carnegie Mellon University
22 * Center for Technology Transfer and Enterprise Creation
23 * 4615 Forbes Avenue
24 * Suite 302
25 * Pittsburgh, PA 15213
26 * (412) 268-7393, fax: (412) 268-7395
27 * innovation@andrew.cmu.edu
28 *
29 * 4. Redistributions of any form whatsoever must retain the following
30 * acknowledgment:
31 * "This product includes software developed by Computing Services
32 * at Carnegie Mellon University (http://www.cmu.edu/computing/)."
33 *
34 * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
35 * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
36 * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
37 * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
38 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
39 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
40 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
41 */
42 #include <config.h>
43
44 #ifdef HAVE_UNISTD_H
45 #include <unistd.h>
46 #endif
47 #include <ctype.h>
48 #include <errno.h>
49 #ifdef HAVE_INTTYPES_H
50 # include <inttypes.h>
51 #elif defined(HAVE_STDINT_H)
52 # include <stdint.h>
53 #endif
54 #include <stdio.h>
55 #include <stdlib.h>
56 #include <string.h>
57 #include <sysexits.h>
58 #include <syslog.h>
59 #include <utime.h>
60
61 #ifdef HAVE_DIRENT_H
62 # include <dirent.h>
63 # define NAMLEN(dirent) strlen((dirent)->d_name)
64 #else
65 # define dirent direct
66 # define NAMLEN(dirent) (dirent)->d_namlen
67 # if HAVE_SYS_NDIR_H
68 # include <sys/ndir.h>
69 # endif
70 # if HAVE_SYS_DIR_H
71 # include <sys/dir.h>
72 # endif
73 # if HAVE_NDIR_H
74 # include <ndir.h>
75 # endif
76 #endif
77
78 #include "annotate.h"
79 #include "assert.h"
80 #ifdef WITH_DAV
81 #include "caldav_db.h"
82 #include "caldav_alarm.h"
83 #include "carddav_db.h"
84 #include "webdav_db.h"
85 #include "ical_support.h"
86 #include "vcard_support.h"
87 #endif /* WITH_DAV */
88 #include "crc32.h"
89 #include "md5.h"
90 #include "global.h"
91 #include "imparse.h"
92 #include "cyr_lock.h"
93 #include "mailbox.h"
94 #include "mappedfile.h"
95 #include "message.h"
96 #include "map.h"
97 #include "mboxevent.h"
98 #include "mboxlist.h"
99 #include "parseaddr.h"
100 #include "proc.h"
101 #include "retry.h"
102 #include "seen.h"
103 #include "util.h"
104 #include "sequence.h"
105 #include "statuscache.h"
106 #include "strarray.h"
107 #include "sync_log.h"
108 #include "xmalloc.h"
109 #include "xstrlcpy.h"
110 #include "xstrlcat.h"
111 #include "xstats.h"
112
113
114 #if defined ENABLE_OBJECTSTORE
115 #include "objectstore.h"
116 #include "objectstore_db.h"
117 #endif
118
119 /* generated headers are not necessarily in current directory */
120 #include "imap/imap_err.h"
121 #include "imap/mailbox_header_cache.h"
122
123 static mailbox_wait_cb_t *mailbox_wait_cb = NULL;
124 static void *mailbox_wait_cb_rock = NULL;
125
126 struct mailboxlist {
127 struct mailboxlist *next;
128 struct mailbox m;
129 struct mboxlock *l;
130 int nopen;
131 };
132
133 static struct mailboxlist *open_mailboxes = NULL;
134
135 #define zeromailbox(m) do { memset(&m, 0, sizeof(struct mailbox)); \
136 (m).index_fd = -1; \
137 (m).header_fd = -1; \
138 } while (0)
139
140 /* for repack */
141 struct mailbox_repack {
142 struct mailbox *mailbox;
143 struct mailbox newmailbox;
144 struct seqset *seqset;
145 struct synccrcs crcs;
146 char *userid;
147 ptrarray_t caches;
148 };
149
150 static struct MsgFlagMap msgflagmap[] = {
151 {"AN", (int)FLAG_ANSWERED},
152 {"FL", (int)FLAG_FLAGGED},
153 {"DE", (int)FLAG_DELETED},
154 {"DR", (int)FLAG_DRAFT},
155 {"SE", (int)FLAG_SEEN},
156 {"SN", (int)FLAG_INTERNAL_SNOOZED},
157 {"SP", (int)FLAG_INTERNAL_SPLITCONVERSATION},
158 {"NC", (int)FLAG_INTERNAL_NEEDS_CLEANUP},
159 {"AR", (int)FLAG_INTERNAL_ARCHIVED},
160 {"UN", (int)FLAG_INTERNAL_UNLINKED},
161 {"EX", (int)FLAG_INTERNAL_EXPUNGED}
162 /* ZZ -> invalid file found on disk */
163 };
164 /* The length of the msgflagmap list * 2 +
165 * Total number of separators possible
166 * = 33 bytes
167 */
168 #define FLAGMAPSTR_MAXLEN (1 + 3 *(sizeof(msgflagmap) / sizeof(struct MsgFlagMap)))
169
170 static int mailbox_index_unlink(struct mailbox *mailbox);
171 static int mailbox_index_repack(struct mailbox *mailbox, int version);
172 static void mailbox_repack_abort(struct mailbox_repack **repackptr);
173 static int mailbox_lock_index_internal(struct mailbox *mailbox,
174 int locktype);
175 static void cleanup_stale_expunged(struct mailbox *mailbox);
176 static bit32 mailbox_index_record_to_buf(struct index_record *record, int version,
177 unsigned char *buf);
178
179 #ifdef WITH_DAV
180 static int mailbox_commit_dav(struct mailbox *mailbox);
181 static int mailbox_abort_dav(struct mailbox *mailbox);
182 static int mailbox_delete_dav(struct mailbox *mailbox);
183 #endif
184
flags_to_str_internal(uint32_t flags,char * flagstr)185 static inline void flags_to_str_internal(uint32_t flags, char *flagstr)
186 {
187 size_t i, len = 0;
188 size_t map_size;
189 char *p;
190
191 p = flagstr;
192 memset(p, 0, FLAGMAPSTR_MAXLEN);
193 map_size = sizeof(msgflagmap) / sizeof(struct MsgFlagMap);
194
195 for (i = 0; i < map_size && len < FLAGMAPSTR_MAXLEN; i++) {
196 if (flags & msgflagmap[i].flag) {
197 if (p != flagstr) {
198 *p++ = '|';
199 len++;
200 }
201
202 *p++ = msgflagmap[i].code[0];
203 *p++ = msgflagmap[i].code[1];
204 len += 2;
205 }
206 }
207 }
208
flags_to_str(struct index_record * record,char * flagstr)209 static void flags_to_str(struct index_record *record, char *flagstr)
210 {
211 uint32_t flags = record->system_flags | record->internal_flags;
212
213 flags_to_str_internal(flags, flagstr);
214 }
215
open_mailboxes_exist()216 EXPORTED int open_mailboxes_exist()
217 {
218 return open_mailboxes ? 1 : 0;
219 }
220
create_listitem(const char * name)221 static struct mailboxlist *create_listitem(const char *name)
222 {
223 struct mailboxlist *item = xmalloc(sizeof(struct mailboxlist));
224 item->next = open_mailboxes;
225 open_mailboxes = item;
226
227 item->nopen = 1;
228 item->l = NULL;
229 zeromailbox(item->m);
230 item->m.name = xstrdup(name);
231 /* ensure we never print insane times */
232 gettimeofday(&item->m.starttime, 0);
233
234 #if defined ENABLE_OBJECTSTORE
235 if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED))
236 keep_user_message_db_open (1);
237 #endif
238
239 return item;
240 }
241
find_listitem(const char * name)242 static struct mailboxlist *find_listitem(const char *name)
243 {
244 struct mailboxlist *item;
245
246 for (item = open_mailboxes; item; item = item->next) {
247 if (!strcmp(name, item->m.name))
248 return item;
249 }
250
251 return NULL;
252 }
253
remove_listitem(struct mailboxlist * remitem)254 static void remove_listitem(struct mailboxlist *remitem)
255 {
256 struct mailboxlist *item;
257 struct mailboxlist *previtem = NULL;
258
259 for (item = open_mailboxes; item; item = item->next) {
260 if (item == remitem) {
261
262 if (previtem)
263 previtem->next = item->next;
264 else
265 open_mailboxes = item->next;
266 free(item);
267
268 #if defined ENABLE_OBJECTSTORE
269 if (!open_mailboxes && config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) // time to close the database
270 keep_user_message_db_open (0);
271 #endif
272 return;
273 }
274 previtem = item;
275 }
276
277 fatal("didn't find item in list", EX_SOFTWARE);
278 }
279
mailbox_meta_fname(struct mailbox * mailbox,int metafile)280 EXPORTED const char *mailbox_meta_fname(struct mailbox *mailbox, int metafile)
281 {
282 static char fnamebuf[MAX_MAILBOX_PATH];
283 const char *src;
284
285 src = mboxname_metapath(mailbox->part, mailbox->name, mailbox->uniqueid, metafile, 0);
286 if (!src) return NULL;
287
288 xstrncpy(fnamebuf, src, MAX_MAILBOX_PATH);
289 return fnamebuf;
290 }
291
mailbox_meta_newfname(struct mailbox * mailbox,int metafile)292 EXPORTED const char *mailbox_meta_newfname(struct mailbox *mailbox, int metafile)
293 {
294 static char fnamebuf[MAX_MAILBOX_PATH];
295 const char *src;
296
297 src = mboxname_metapath(mailbox->part, mailbox->name, mailbox->uniqueid, metafile, 1);
298 if (!src) return NULL;
299
300 xstrncpy(fnamebuf, src, MAX_MAILBOX_PATH);
301 return fnamebuf;
302 }
303
mailbox_meta_rename(struct mailbox * mailbox,int metafile)304 EXPORTED int mailbox_meta_rename(struct mailbox *mailbox, int metafile)
305 {
306 const char *fname = mailbox_meta_fname(mailbox, metafile);
307 const char *newfname = mailbox_meta_newfname(mailbox, metafile);
308
309 if (rename(newfname, fname))
310 return IMAP_IOERROR;
311
312 return 0;
313 }
314
mailbox_spool_fname(struct mailbox * mailbox,uint32_t uid)315 static const char *mailbox_spool_fname(struct mailbox *mailbox, uint32_t uid)
316 {
317 return mboxname_datapath(mailbox->part, mailbox->name, mailbox->uniqueid, uid);
318 }
319
mailbox_archive_fname(struct mailbox * mailbox,uint32_t uid)320 static const char *mailbox_archive_fname(struct mailbox *mailbox, uint32_t uid)
321 {
322 return mboxname_archivepath(mailbox->part, mailbox->name, mailbox->uniqueid, uid);
323 }
324
mailbox_record_fname(struct mailbox * mailbox,const struct index_record * record)325 EXPORTED const char *mailbox_record_fname(struct mailbox *mailbox,
326 const struct index_record *record)
327 {
328 int object_storage_enabled = 0 ;
329 #if defined ENABLE_OBJECTSTORE
330 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
331 #endif
332
333 if (!object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))
334 return mailbox_archive_fname(mailbox, record->uid);
335 else
336 return mailbox_spool_fname(mailbox, record->uid);
337 }
338
mailbox_datapath(struct mailbox * mailbox,uint32_t uid)339 EXPORTED const char *mailbox_datapath(struct mailbox *mailbox, uint32_t uid)
340 {
341 static char localbuf[MAX_MAILBOX_PATH];
342 const char *src;
343
344 src = mboxname_datapath(mailbox->part, mailbox->name, mailbox->uniqueid, uid);
345 if (!src) return NULL;
346
347 xstrncpy(localbuf, src, MAX_MAILBOX_PATH);
348 return localbuf;
349 }
350
351 /*
352 * Function to test if a header is in the cache
353 *
354 * Assume cache entry version 1, unless other data is found
355 * in the table.
356 */
is_cached_header(const char * hdr)357 static inline unsigned is_cached_header(const char *hdr)
358 {
359 size_t len;
360 struct mailbox_header_cache *thdr;
361
362 len = strlen(hdr);
363 if (len >= MAX_CACHED_HEADER_SIZE)
364 return BIT32_MAX;
365
366 thdr = mailbox_header_cache_lookup(hdr, len);
367
368 if (thdr)
369 return thdr->min_cache_version;
370
371 /* Don't Cache X- headers unless explicitly configured to*/
372 if (((hdr[0] == 'x') || (hdr[0] == 'X')) &&
373 (hdr[1] == '-')) return BIT32_MAX;
374
375 /* Everything else we cache in version 1 */
376 return 1;
377 }
378
379 /* External API to is_cached_header that prepares the string
380 *
381 * Returns minimum version required for lookup to succeed
382 * or BIT32_MAX if header not cached
383 */
mailbox_cached_header(const char * s)384 EXPORTED unsigned mailbox_cached_header(const char *s)
385 {
386 return is_cached_header(s);
387 }
388
389 /* Same as mailbox_cached_header, but for use on a header
390 * as it appears in the message (i.e. :-terminated, not NUL-terminated)
391 */
mailbox_cached_header_inline(const char * text)392 HIDDEN unsigned mailbox_cached_header_inline(const char *text)
393 {
394 char buf[MAX_CACHED_HEADER_SIZE];
395 int i;
396
397 /* Scan for header */
398 for (i=0; i < MAX_CACHED_HEADER_SIZE; i++) {
399 if (!text[i] || text[i] == '\r' || text[i] == '\n') break;
400
401 if (text[i] == ':') {
402 buf[i] = '\0';
403 return is_cached_header(buf);
404 } else {
405 buf[i] = text[i];
406 }
407 }
408
409 return BIT32_MAX;
410 }
411
cache_base(const struct index_record * record)412 static const char *cache_base(const struct index_record *record)
413 {
414 const char *base = record->crec.buf->s;
415 return base + record->crec.offset;
416 }
417
cache_len(const struct index_record * record)418 static size_t cache_len(const struct index_record *record)
419 {
420 return record->crec.len;
421 }
422
cache_buf(const struct index_record * record)423 static struct buf *cache_buf(const struct index_record *record)
424 {
425 static struct buf staticbuf;
426
427 buf_init_ro(&staticbuf,
428 cache_base(record),
429 cache_len(record));
430
431 return &staticbuf;
432 }
433
cacheitem_base(const struct index_record * record,int field)434 EXPORTED const char *cacheitem_base(const struct index_record *record, int field)
435 {
436 const char *base = record->crec.buf->s;
437 return base + record->crec.item[field].offset;
438 }
439
cacheitem_size(const struct index_record * record,int field)440 EXPORTED unsigned cacheitem_size(const struct index_record *record, int field)
441 {
442 return record->crec.item[field].len;
443 }
444
cacheitem_buf(const struct index_record * record,int field)445 EXPORTED struct buf *cacheitem_buf(const struct index_record *record, int field)
446 {
447 static struct buf staticbuf;
448
449 buf_init_ro(&staticbuf,
450 cacheitem_base(record, field),
451 cacheitem_size(record, field));
452
453 return &staticbuf;
454 }
455
456 /* parse a single cache record from the mapped file - creates buf
457 * records which point into the map, so you can't free it while
458 * you still have them around! */
cache_parserecord(struct mappedfile * cachefile,uint64_t cache_offset,struct cacherecord * crec)459 static int cache_parserecord(struct mappedfile *cachefile, uint64_t cache_offset,
460 struct cacherecord *crec)
461 {
462 const struct buf *buf = mappedfile_buf(cachefile);
463 size_t buf_size = mappedfile_size(cachefile);
464 const char *cacheitem, *next;
465 size_t offset;
466 int cache_ent;
467
468 offset = cache_offset;
469
470 for (cache_ent = 0; cache_ent < NUM_CACHE_FIELDS; cache_ent++) {
471 cacheitem = buf->s + offset;
472
473 /* bounds checking */
474 if (offset >= buf_size) {
475 syslog(LOG_ERR, "IOERROR: offset greater than cache size "
476 SIZE_T_FMT " " SIZE_T_FMT "(%d)",
477 offset, buf_size, cache_ent);
478 return IMAP_IOERROR;
479 }
480
481 if (offset + CACHE_ITEM_SIZE_SKIP + CACHE_ITEM_LEN(cacheitem) > buf_size) {
482 syslog(LOG_ERR, "IOERROR: cache entry truncated "
483 SIZE_T_FMT " %u " SIZE_T_FMT "(%d)",
484 offset, CACHE_ITEM_LEN(cacheitem),
485 buf_size, cache_ent);
486 return IMAP_IOERROR;
487 }
488
489 /* copy locations */
490 crec->item[cache_ent].len = CACHE_ITEM_LEN(cacheitem);
491 crec->item[cache_ent].offset = offset + CACHE_ITEM_SIZE_SKIP;
492
493 /* moving on */
494 next = CACHE_ITEM_NEXT(cacheitem);
495 if (next < cacheitem) {
496 syslog(LOG_ERR, "IOERROR: cache offset negative");
497 return IMAP_IOERROR;
498 }
499
500 offset = next - buf->s;
501 }
502
503 /* all fit within the cache, it's gold as far as we can tell */
504 crec->buf = buf;
505 crec->len = offset - cache_offset;
506 crec->offset = cache_offset;
507
508 return 0;
509 }
510
mailbox_cache_get_env(struct mailbox * mailbox,const struct index_record * record,int token)511 EXPORTED char *mailbox_cache_get_env(struct mailbox *mailbox,
512 const struct index_record *record,
513 int token)
514 {
515 char *env;
516 char *envtokens[NUMENVTOKENS];
517 char *field;
518
519 if (mailbox_cacherecord(mailbox, record))
520 return NULL;
521
522 if (cacheitem_size(record, CACHE_ENVELOPE) <= 2)
523 return NULL;
524
525 /* get field out of the envelope
526 *
527 * get a working copy; strip outer ()'s
528 * +1 -> skip the leading paren
529 * -2 -> don't include the size of the outer parens
530 */
531 env = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1,
532 cacheitem_size(record, CACHE_ENVELOPE) - 2);
533 parse_cached_envelope(env, envtokens, VECTOR_SIZE(envtokens));
534
535 field = xstrdupnull(envtokens[token]);
536
537 /* free stuff */
538 free(env);
539
540 return field;
541 }
542
mailbox_index_islocked(struct mailbox * mailbox,int write)543 EXPORTED int mailbox_index_islocked(struct mailbox *mailbox, int write)
544 {
545 if (mailbox->index_locktype == LOCK_EXCLUSIVE) return 1;
546 if (mailbox->index_locktype == LOCK_SHARED && !write) return 1;
547 return 0;
548 }
549
cache_append_record(struct mappedfile * mf,struct index_record * record)550 static int cache_append_record(struct mappedfile *mf, struct index_record *record)
551 {
552 const struct buf *buf = cache_buf(record);
553 size_t offset = mappedfile_size(mf);
554 int n;
555
556 n = mappedfile_pwritebuf(mf, buf, offset);
557 if (n < 0) {
558 syslog(LOG_ERR, "failed to append " SIZE_T_FMT " bytes to cache", buf->len);
559 return IMAP_IOERROR;
560 }
561
562 record->cache_offset = offset;
563
564 return 0;
565 }
566
cache_getfile(ptrarray_t * list,const char * fname,int readonly,uint32_t generation)567 static struct mappedfile *cache_getfile(ptrarray_t *list, const char *fname,
568 int readonly, uint32_t generation)
569 {
570 struct mappedfile *cachefile = NULL;
571 int openflags = readonly ? 0 : MAPPEDFILE_CREATE | MAPPEDFILE_RW;
572 int i;
573
574 for (i = 0; i < list->count; i++) {
575 cachefile = ptrarray_nth(list, i);
576 if (!strcmp(fname, mappedfile_fname(cachefile)))
577 return cachefile;
578 }
579
580 /* guess we didn't find it - open a new one */
581 cachefile = NULL;
582 if (mappedfile_open(&cachefile, fname, openflags)) {
583 syslog(LOG_ERR, "IOERROR: failed to open cache file %s", fname);
584 return NULL;
585 }
586
587 if (!readonly && !mappedfile_size(cachefile)) {
588 /* zero byte file? Set the generation */
589 uint32_t netgen = htonl(generation);
590 mappedfile_pwrite(cachefile, (const char *) &netgen, 4, 0);
591 mappedfile_commit(cachefile);
592 }
593
594 ptrarray_append(list, cachefile);
595
596 return cachefile;
597 }
598
mailbox_cachefile(struct mailbox * mailbox,const struct index_record * record)599 static struct mappedfile *mailbox_cachefile(struct mailbox *mailbox,
600 const struct index_record *record)
601 {
602 const char *fname;
603
604 if (record->internal_flags & FLAG_INTERNAL_ARCHIVED)
605 fname = mailbox_meta_fname(mailbox, META_ARCHIVECACHE);
606 else
607 fname = mailbox_meta_fname(mailbox, META_CACHE);
608
609 return cache_getfile(&mailbox->caches, fname, mailbox->is_readonly, mailbox->i.generation_no);
610 }
611
repack_cachefile(struct mailbox_repack * repack,const struct index_record * record)612 static struct mappedfile *repack_cachefile(struct mailbox_repack *repack,
613 const struct index_record *record)
614 {
615 const char *fname;
616
617 if (record->internal_flags & FLAG_INTERNAL_ARCHIVED)
618 fname = mailbox_meta_newfname(repack->mailbox, META_ARCHIVECACHE);
619 else
620 fname = mailbox_meta_newfname(repack->mailbox, META_CACHE);
621
622 return cache_getfile(&repack->caches, fname, /*readonly*/0, repack->newmailbox.i.generation_no);
623 }
624
625 /* return the offset for the start of the record! */
mailbox_append_cache(struct mailbox * mailbox,struct index_record * record)626 static int mailbox_append_cache(struct mailbox *mailbox,
627 struct index_record *record)
628 {
629 struct mappedfile *cachefile;
630 int r;
631
632 assert(mailbox_index_islocked(mailbox, 1));
633
634 /* already been written */
635 if (record->cache_offset)
636 return 0;
637
638 /* no cache content */
639 if (!record->crec.len) {
640 /* make one! */
641 const char *fname = mailbox_record_fname(mailbox, record);
642 syslog(LOG_ERR, "IOERROR: no cache for %s %u, parsing and saving",
643 mailbox->name, record->uid);
644 r = message_parse(fname, record);
645 if (r) return r;
646 mailbox_index_dirty(mailbox);
647 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
648 }
649
650 cachefile = mailbox_cachefile(mailbox, record);
651 if (!cachefile) {
652 syslog(LOG_ERR, "Failed to open cache to %s for %u",
653 mailbox->name, record->uid);
654 return IMAP_IOERROR; /* unable to append */
655 }
656
657 r = cache_append_record(cachefile, record);
658 if (r) {
659 syslog(LOG_ERR, "Failed to append cache to %s for %u",
660 mailbox->name, record->uid);
661 return r;
662 }
663
664 return 0;
665 }
666
667 enum mbcache_rewrite_t {
668 MBCACHE_NOPARSE,
669 MBCACHE_PARSEONLY,
670 MBCACHE_REWRITE
671 };
672
mailbox_cacherecord_internal(struct mailbox * mailbox,const struct index_record * record,enum mbcache_rewrite_t rewrite)673 static int mailbox_cacherecord_internal(struct mailbox *mailbox,
674 const struct index_record *record,
675 enum mbcache_rewrite_t rewrite)
676 {
677 struct mappedfile *cachefile;
678 bit32 crc = 0;
679 int r = IMAP_IOERROR;
680
681 /* we do something nasty here to work around lazy loading while still
682 * giving const protection to records which are only used for read */
683 struct index_record *backdoor = (struct index_record *)record;
684
685 /* do we already have a record loaded? */
686 if (record->crec.len)
687 return 0;
688
689 /* make sure there's a file to read from */
690 cachefile = mailbox_cachefile(mailbox, record);
691 if (!cachefile)
692 goto err;
693
694 /* do we have an offset? */
695 if (!record->cache_offset)
696 goto err;
697
698 /* try to parse the cache record */
699 r = cache_parserecord(cachefile, record->cache_offset, &backdoor->crec);
700 if (r) goto err;
701
702 /* old-style record */
703 if (!record->cache_crc)
704 goto err;
705
706 crc = crc32_buf(cache_buf(record));
707 if (crc != record->cache_crc)
708 r = IMAP_MAILBOX_CHECKSUM;
709
710 if (r) goto err;
711 return 0;
712
713 err:
714 if (!cachefile)
715 syslog(LOG_ERR, "IOERROR: missing cache file for %s uid %u",
716 mailbox->name, record->uid);
717 else if (!record->cache_offset)
718 syslog(LOG_ERR, "IOERROR: missing cache offset for %s uid %u",
719 mailbox->name, record->uid);
720 else if (r)
721 syslog(LOG_ERR, "IOERROR: invalid cache record for %s uid %u (%s) %d at %llu",
722 mailbox->name, record->uid, error_message(r), crc, (long long unsigned)record->cache_offset);
723
724 if (rewrite == MBCACHE_NOPARSE)
725 return r;
726
727 /* parse the file again */
728
729 /* parse directly into the cache for this record */
730 const char *fname = mailbox_record_fname(mailbox, record);
731 if (!fname) {
732 syslog(LOG_ERR, "IOERROR: no spool file for %s uid %u",
733 mailbox->name, record->uid);
734 return IMAP_IOERROR;
735 }
736
737 /* parse into the file and zero the cache offset */
738 r = message_parse(fname, backdoor);
739 if (r) {
740 syslog(LOG_ERR, "IOERROR: failed to parse message for %s uid %u",
741 mailbox->name, record->uid);
742 return r;
743 }
744 backdoor->cache_offset = 0;
745
746 if (rewrite == MBCACHE_REWRITE) {
747 r = mailbox_append_cache(mailbox, backdoor);
748 if (r) {
749 syslog(LOG_ERR, "IOERROR: failed to append cache for %s uid %u",
750 mailbox->name, record->uid);
751 return r;
752 }
753 }
754
755 return 0;
756 }
757
mailbox_cacherecord(struct mailbox * mailbox,const struct index_record * record)758 EXPORTED int mailbox_cacherecord(struct mailbox *mailbox, const struct index_record *record)
759 {
760 enum mbcache_rewrite_t rewrite = mailbox_index_islocked(mailbox, 1) ? MBCACHE_REWRITE : MBCACHE_PARSEONLY;
761 return mailbox_cacherecord_internal(mailbox, record, rewrite);
762 }
763
mailbox_abort_cache(struct mailbox * mailbox)764 static int mailbox_abort_cache(struct mailbox *mailbox)
765 {
766 if (!mailbox->caches.count)
767 return 0;
768
769 /* XXX - we need offsets into each file to use
770 mappedfile_truncate */
771
772 return 0;
773 }
774
mailbox_commit_cache(struct mailbox * mailbox)775 static int mailbox_commit_cache(struct mailbox *mailbox)
776 {
777 int i;
778
779 for (i = 0; i < mailbox->caches.count; i++) {
780 struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i);
781 int r = mappedfile_commit(cachefile);
782 if (r) return r;
783 }
784
785 return 0;
786 }
787
788 /* function to be used for notification of mailbox changes/updates */
789 static mailbox_notifyproc_t *updatenotifier = NULL;
790
791 /*
792 * Set the updatenotifier function
793 */
mailbox_set_updatenotifier(mailbox_notifyproc_t * notifyproc)794 HIDDEN void mailbox_set_updatenotifier(mailbox_notifyproc_t *notifyproc)
795 {
796 updatenotifier = notifyproc;
797 }
798
799 /*
800 * Get the updatenotifier function
801 */
mailbox_get_updatenotifier(void)802 mailbox_notifyproc_t *mailbox_get_updatenotifier(void)
803 {
804 return updatenotifier;
805 }
806
mailbox_set_uniqueid(struct mailbox * mailbox,const char * uniqueid)807 static void mailbox_set_uniqueid(struct mailbox *mailbox, const char *uniqueid)
808 {
809 free(mailbox->uniqueid);
810 mailbox->uniqueid = xstrdup(uniqueid);
811 mailbox->header_dirty = 1;
812 }
813
814 /*
815 * Create the unique identifier for a mailbox named 'name' with
816 * uidvalidity 'uidvalidity'. We use Ted Ts'o's libuuid if available,
817 * otherwise we use some random bits.
818 */
mailbox_make_uniqueid(struct mailbox * mailbox)819 EXPORTED void mailbox_make_uniqueid(struct mailbox *mailbox)
820 {
821 mailbox_set_uniqueid(mailbox, makeuuid());
822 }
823
_map_local_record(const struct mailbox * mailbox,const char * fname,struct buf * buf)824 static int _map_local_record(const struct mailbox *mailbox, const char *fname, struct buf *buf)
825 {
826 struct stat sbuf;
827 int msgfd;
828
829 msgfd = open(fname, O_RDONLY, 0666);
830 if (msgfd == -1) return errno;
831
832 if (fstat(msgfd, &sbuf) == -1) {
833 syslog(LOG_ERR, "IOERROR: fstat on %s: %m", fname);
834 fatal("can't fstat message file", EX_OSFILE);
835 }
836
837 buf_refresh_mmap(buf, /*onceonly*/1, msgfd, fname, sbuf.st_size, mailbox->name);
838 close(msgfd);
839
840 return 0;
841 }
842
mailbox_map_record(struct mailbox * mailbox,const struct index_record * record,struct buf * buf)843 EXPORTED int mailbox_map_record(struct mailbox *mailbox, const struct index_record *record, struct buf *buf)
844 {
845 xstats_inc(MESSAGE_MAP);
846
847 const char *fname = mailbox_record_fname(mailbox, record);
848 int r = _map_local_record(mailbox, fname, buf);
849 if (!r) return 0;
850
851 #if defined ENABLE_OBJECTSTORE
852 if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)){
853 r = objectstore_get(mailbox, record, fname);
854 if (r) return r;
855 r = _map_local_record(mailbox, fname, buf);
856 remove(fname);
857 }
858 #endif
859
860 return r;
861 }
862
863
mailbox_release_resources(struct mailbox * mailbox)864 static void mailbox_release_resources(struct mailbox *mailbox)
865 {
866 int i;
867
868 if (mailbox->i.dirty)
869 abort();
870
871 /* just close the header */
872 xclose(mailbox->header_fd);
873
874 /* release and unmap index */
875 xclose(mailbox->index_fd);
876 mailbox->index_locktype = 0; /* lock was released by closing fd */
877 if (mailbox->index_base)
878 map_free(&mailbox->index_base, &mailbox->index_len);
879
880 /* release caches */
881 for (i = 0; i < mailbox->caches.count; i++) {
882 struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i);
883 mappedfile_close(&cachefile);
884 }
885 ptrarray_fini(&mailbox->caches);
886 }
887
888 /*
889 * Open the index file for 'mailbox'
890 */
mailbox_open_index(struct mailbox * mailbox)891 static int mailbox_open_index(struct mailbox *mailbox)
892 {
893 struct stat sbuf;
894 const char *fname;
895 int openflags = mailbox->is_readonly ? O_RDONLY : O_RDWR;
896
897 mailbox_release_resources(mailbox);
898
899 /* open and map the index file */
900 fname = mailbox_meta_fname(mailbox, META_INDEX);
901 if (!fname)
902 return IMAP_MAILBOX_BADNAME;
903
904 mailbox->index_fd = open(fname, openflags, 0);
905 if (mailbox->index_fd == -1)
906 return IMAP_IOERROR;
907
908 /* don't open the cache yet, it will be loaded by lazy-loading
909 * later */
910
911 fstat(mailbox->index_fd, &sbuf);
912 mailbox->index_ino = sbuf.st_ino;
913 mailbox->index_mtime = sbuf.st_mtime;
914 mailbox->index_size = sbuf.st_size;
915 map_refresh(mailbox->index_fd, 0, &mailbox->index_base,
916 &mailbox->index_len, mailbox->index_size,
917 "index", mailbox->name);
918
919 return 0;
920 }
921
mailbox_mboxlock_reopen(struct mailboxlist * listitem,int locktype)922 static int mailbox_mboxlock_reopen(struct mailboxlist *listitem, int locktype)
923 {
924 struct mailbox *mailbox = &listitem->m;
925 int r;
926
927 mailbox_release_resources(mailbox);
928
929 mboxname_release(&listitem->l);
930 r = mboxname_lock(mailbox->name, &listitem->l, locktype);
931 if (r) return r;
932
933 return r;
934 }
935
936 /*
937 * Open and read the header of the mailbox with name 'name'
938 * The structure pointed to by 'mailbox' is initialized.
939 */
mailbox_open_advanced(const char * name,int locktype,int index_locktype,struct mailbox ** mailboxptr)940 static int mailbox_open_advanced(const char *name,
941 int locktype,
942 int index_locktype,
943 struct mailbox **mailboxptr)
944 {
945 mbentry_t *mbentry = NULL;
946 struct mailboxlist *listitem;
947 struct mailbox *mailbox = NULL;
948 int r = 0;
949
950 assert(*mailboxptr == NULL);
951
952 listitem = find_listitem(name);
953
954 /* already open? just use this one */
955 if (listitem) {
956 /* can't reuse an exclusive locked mailbox */
957 if (listitem->l->locktype == LOCK_EXCLUSIVE)
958 return IMAP_MAILBOX_LOCKED;
959 if (locktype == LOCK_EXCLUSIVE)
960 return IMAP_MAILBOX_LOCKED;
961 /* can't reuse an already locked index */
962 if (listitem->m.index_locktype)
963 return IMAP_MAILBOX_LOCKED;
964
965 listitem->nopen++;
966 mailbox = &listitem->m;
967
968 goto lockindex;
969 }
970
971 listitem = create_listitem(name);
972 mailbox = &listitem->m;
973
974 r = mboxname_lock(name, &listitem->l, locktype);
975 if (r) {
976 /* locked is not an error - just means we asked for NONBLOCKING */
977 if (r != IMAP_MAILBOX_LOCKED)
978 syslog(LOG_ERR, "IOERROR: locking %s: %s",
979 mailbox->name, error_message(r));
980 goto done;
981 }
982
983 r = mboxlist_lookup_allow_all(name, &mbentry, NULL);
984 if (r) goto done;
985
986 if (mbentry->mbtype & MBTYPE_MOVING) {
987 mboxlist_entry_free(&mbentry);
988 r = IMAP_MAILBOX_MOVED;
989 goto done;
990 }
991
992 if (!mbentry->partition) {
993 mboxlist_entry_free(&mbentry);
994 r = IMAP_MAILBOX_NONEXISTENT;
995 goto done;
996 }
997
998 mailbox->part = xstrdup(mbentry->partition);
999
1000 /* Note that the header does have the ACL information, but it is only
1001 * a backup, and the mboxlist data is considered authoritative, so
1002 * we will just use what we were passed */
1003 mailbox->acl = xstrdup(mbentry->acl);
1004 mailbox->mbtype = mbentry->mbtype;
1005 mailbox->foldermodseq = mbentry->foldermodseq;
1006
1007 mboxlist_entry_free(&mbentry);
1008
1009 if (index_locktype == LOCK_SHARED)
1010 mailbox->is_readonly = 1;
1011
1012 r = mailbox_open_index(mailbox);
1013 if (r) {
1014 syslog(LOG_ERR, "IOERROR: opening index %s: %s",
1015 mailbox->name, error_message(r));
1016 goto done;
1017 }
1018
1019 lockindex:
1020 r = mailbox_lock_index(mailbox, index_locktype);
1021
1022 /* we always nuke expunged if the version is less than 12 */
1023 if (mailbox->i.minor_version < 12)
1024 cleanup_stale_expunged(mailbox);
1025
1026 done:
1027 if (r) mailbox_close(&mailbox);
1028 else *mailboxptr = mailbox;
1029
1030 return r;
1031 }
1032
mailbox_open_irl(const char * name,struct mailbox ** mailboxptr)1033 EXPORTED int mailbox_open_irl(const char *name, struct mailbox **mailboxptr)
1034 {
1035 return mailbox_open_advanced(name, LOCK_SHARED, LOCK_SHARED,
1036 mailboxptr);
1037 }
1038
mailbox_open_iwl(const char * name,struct mailbox ** mailboxptr)1039 EXPORTED int mailbox_open_iwl(const char *name, struct mailbox **mailboxptr)
1040 {
1041 return mailbox_open_advanced(name, LOCK_SHARED, LOCK_EXCLUSIVE,
1042 mailboxptr);
1043 }
1044
mailbox_open_irlnb(const char * name,struct mailbox ** mailboxptr)1045 EXPORTED int mailbox_open_irlnb(const char *name, struct mailbox **mailboxptr)
1046 {
1047 return mailbox_open_advanced(name,
1048 LOCK_SHARED|LOCK_NONBLOCK,
1049 /* cannot do nonblocking lock on index...why? */
1050 LOCK_SHARED,
1051 mailboxptr);
1052 }
1053
mailbox_open_exclusive(const char * name,struct mailbox ** mailboxptr)1054 EXPORTED int mailbox_open_exclusive(const char *name, struct mailbox **mailboxptr)
1055 {
1056 return mailbox_open_advanced(name, LOCK_EXCLUSIVE, LOCK_EXCLUSIVE,
1057 mailboxptr);
1058 }
1059
mailbox_index_dirty(struct mailbox * mailbox)1060 EXPORTED void mailbox_index_dirty(struct mailbox *mailbox)
1061 {
1062 assert(mailbox_index_islocked(mailbox, 1));
1063 mailbox->i.dirty = 1;
1064 }
1065
mailbox_modseq_dirty(struct mailbox * mailbox)1066 EXPORTED modseq_t mailbox_modseq_dirty(struct mailbox *mailbox)
1067 {
1068 assert(mailbox_index_islocked(mailbox, 1));
1069
1070 if (mailbox->silentchanges)
1071 return mailbox->i.highestmodseq;
1072
1073 if (!mailbox->modseq_dirty) {
1074 mailbox->i.highestmodseq = mboxname_setmodseq(mailbox->name,
1075 mailbox->i.highestmodseq,
1076 mailbox->mbtype, /*dofolder*/0);
1077 mailbox->last_updated = time(0);
1078 mailbox->modseq_dirty = 1;
1079 mailbox_index_dirty(mailbox);
1080 }
1081
1082 mailbox->i.highestmodseq++;
1083
1084 return mailbox->i.highestmodseq;
1085 }
1086
mailbox_setversion(struct mailbox * mailbox,int version)1087 EXPORTED int mailbox_setversion(struct mailbox *mailbox, int version)
1088 {
1089 int r = 0;
1090
1091 if (version && mailbox->i.minor_version != version) {
1092 /* need to re-set the version! */
1093 struct mailboxlist *listitem = find_listitem(mailbox->name);
1094
1095 assert(listitem);
1096 assert(&listitem->m == mailbox);
1097
1098 /* we need an exclusive lock on the listitem because we're renaming
1099 * index files, so release locks and then go full exclusive */
1100 mailbox_unlock_index(mailbox, NULL);
1101 r = mailbox_mboxlock_reopen(listitem, LOCK_EXCLUSIVE);
1102
1103 /* we need to re-open the index because we dropped the mboxname lock,
1104 * so the file may have changed */
1105 if (!r) r = mailbox_open_index(mailbox);
1106
1107 /* lock_internal so DELETED doesn't cause it to appear to be
1108 * NONEXISTENT */
1109 if (!r) r = mailbox_lock_index_internal(mailbox, LOCK_EXCLUSIVE);
1110
1111 /* perform the actual repack! */
1112 if (!r) r = mailbox_index_repack(mailbox, version);
1113
1114 /* NOTE: this leaves the mailbox in an unlocked state internally, so
1115 * let's release all the acutal locks */
1116 mailbox_unlock_index(mailbox, NULL);
1117
1118 /* we're also still holding an exclusive namelock in the listitem,
1119 * but that's OK because the only caller will be calling mailbox_close
1120 * immediately afterwards */
1121 }
1122
1123 return r;
1124 }
1125
1126 /*
1127 * Close the mailbox 'mailbox', freeing all associated resources.
1128 */
mailbox_close(struct mailbox ** mailboxptr)1129 EXPORTED void mailbox_close(struct mailbox **mailboxptr)
1130 {
1131 int flag;
1132 struct mailbox *mailbox = *mailboxptr;
1133 struct mailboxlist *listitem;
1134
1135 /* be safe against double-close */
1136 if (!mailbox) return;
1137
1138 listitem = find_listitem(mailbox->name);
1139 assert(listitem && &listitem->m == mailbox);
1140
1141 *mailboxptr = NULL;
1142
1143 /* open multiple times? Just close this one */
1144 if (listitem->nopen > 1) {
1145 listitem->nopen--;
1146 mailbox_unlock_index(mailbox, NULL);
1147 return;
1148 }
1149
1150 /* get a re-read of the options field for cleanup purposes */
1151 if (mailbox->index_fd != -1) {
1152 if (!mailbox->index_locktype)
1153 mailbox_lock_index(mailbox, LOCK_SHARED);
1154 /* drop the index lock here because we'll lose our right to it
1155 * when try to upgrade the mboxlock anyway. */
1156 mailbox_unlock_index(mailbox, NULL);
1157 }
1158
1159 /* do we need to try and clean up? (not if doing a shutdown,
1160 * speed is probably more important!) */
1161 if (!in_shutdown && (mailbox->i.options & MAILBOX_CLEANUP_MASK)) {
1162 int r = mailbox_mboxlock_reopen(listitem, LOCK_NONBLOCKING);
1163 /* we need to re-open the index because we dropped the mboxname lock,
1164 * so the file may have changed */
1165 if (!r) r = mailbox_open_index(mailbox);
1166 /* lock_internal so DELETED doesn't cause it to appear to be
1167 * NONEXISTENT */
1168 if (!r) r = mailbox_lock_index_internal(mailbox, LOCK_EXCLUSIVE);
1169 if (!r) {
1170 /* finish cleaning up */
1171 if (mailbox->i.options & OPT_MAILBOX_DELETED)
1172 mailbox_delete_cleanup(mailbox, mailbox->part, mailbox->name, mailbox->uniqueid);
1173 else if (mailbox->i.options & OPT_MAILBOX_NEEDS_REPACK)
1174 mailbox_index_repack(mailbox, mailbox->i.minor_version);
1175 else if (mailbox->i.options & OPT_MAILBOX_NEEDS_UNLINK)
1176 mailbox_index_unlink(mailbox);
1177 /* or we missed out - someone else beat us to it */
1178
1179 /* anyway, unlock again */
1180 mailbox_unlock_index(mailbox, NULL);
1181 }
1182 /* otherwise someone else has the mailbox locked
1183 * already, so they can handle the cleanup in
1184 * THEIR mailbox_close call */
1185 }
1186
1187 mailbox_release_resources(mailbox);
1188
1189 free(mailbox->name);
1190 free(mailbox->part);
1191 free(mailbox->acl);
1192 free(mailbox->uniqueid);
1193 free(mailbox->quotaroot);
1194
1195 for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
1196 free(mailbox->flagname[flag]);
1197 }
1198
1199 if (listitem->l) mboxname_release(&listitem->l);
1200
1201 remove_listitem(listitem);
1202 }
1203
1204 /*
1205 * Read the header of 'mailbox'
1206 * format:
1207 * MAGIC
1208 * quotaroot TAB uniqueid
1209 * userflag1 SPACE userflag2 SPACE userflag3 [...] (with no trailing space)
1210 * user1 TAB user1acl TAB user2 TAB user2acl TAB (with trailing tab!)
1211 */
mailbox_read_header(struct mailbox * mailbox,char ** aclptr)1212 static int mailbox_read_header(struct mailbox *mailbox, char **aclptr)
1213 {
1214 int r = 0;
1215 int flag;
1216 const char *name, *p, *tab, *eol;
1217 const char *fname;
1218 struct stat sbuf;
1219 const char *base = NULL;
1220 size_t len = 0;
1221 unsigned magic_size = sizeof(MAILBOX_HEADER_MAGIC) - 1;
1222
1223 /* can't be dirty if we're reading it */
1224 if (mailbox->header_dirty)
1225 abort();
1226
1227 xclose(mailbox->header_fd);
1228
1229 fname = mailbox_meta_fname(mailbox, META_HEADER);
1230 mailbox->header_fd = open(fname, O_RDONLY, 0);
1231
1232 if (mailbox->header_fd == -1) {
1233 r = IMAP_IOERROR;
1234 goto done;
1235 }
1236
1237 if (fstat(mailbox->header_fd, &sbuf) == -1) {
1238 xclose(mailbox->header_fd);
1239 r = IMAP_IOERROR;
1240 goto done;
1241 }
1242
1243 map_refresh(mailbox->header_fd, 1, &base, &len,
1244 sbuf.st_size, "header", mailbox->name);
1245 mailbox->header_file_ino = sbuf.st_ino;
1246 mailbox->header_file_crc = crc32_map(base, sbuf.st_size);
1247
1248 /* Check magic number */
1249 if ((unsigned) sbuf.st_size < magic_size ||
1250 strncmp(base, MAILBOX_HEADER_MAGIC, magic_size)) {
1251 r = IMAP_MAILBOX_BADFORMAT;
1252 goto done;
1253 }
1254
1255 /* Read quota data line */
1256 p = base + sizeof(MAILBOX_HEADER_MAGIC)-1;
1257 tab = memchr(p, '\t', sbuf.st_size - (p - base));
1258 eol = memchr(p, '\n', sbuf.st_size - (p - base));
1259 if (!eol) {
1260 r = IMAP_MAILBOX_BADFORMAT;
1261 goto done;
1262 }
1263
1264 /* quotaroot (if present) */
1265 free(mailbox->quotaroot);
1266 if (!tab || tab > eol) {
1267 syslog(LOG_DEBUG, "mailbox '%s' has old cyrus.header",
1268 mailbox->name);
1269 tab = eol;
1270 }
1271 if (p < tab) {
1272 mailbox->quotaroot = xstrndup(p, tab - p);
1273 }
1274 else {
1275 mailbox->quotaroot = NULL;
1276 }
1277
1278 /* read uniqueid (should always exist unless old format) */
1279 free(mailbox->uniqueid);
1280 mailbox->uniqueid = NULL;
1281 if (tab < eol) {
1282 p = tab + 1;
1283 if (p == eol) {
1284 r = IMAP_MAILBOX_BADFORMAT;
1285 goto done;
1286 }
1287 tab = memchr(p, '\t', sbuf.st_size - (p - base));
1288 if (!tab || tab > eol) tab = eol;
1289 mailbox->uniqueid = xstrndup(p, tab - p);
1290 }
1291 /* else, uniqueid needs to be generated when we know the uidvalidity */
1292
1293 /* Read names of user flags */
1294 p = eol + 1;
1295 eol = memchr(p, '\n', sbuf.st_size - (p - base));
1296 if (!eol) {
1297 r = IMAP_MAILBOX_BADFORMAT;
1298 goto done;
1299 }
1300 name = p;
1301 /* read the names of flags */
1302 for (flag = 0; name <= eol && flag < MAX_USER_FLAGS; flag++) {
1303 free(mailbox->flagname[flag]);
1304 mailbox->flagname[flag] = NULL;
1305 p = memchr(name, ' ', eol-name);
1306 if (!p) p = eol;
1307 if (name != p)
1308 mailbox->flagname[flag] = xstrndup(name, p-name);
1309 name = p+1;
1310 }
1311 /* zero out the rest */
1312 for (; flag < MAX_USER_FLAGS; flag++) {
1313 free(mailbox->flagname[flag]);
1314 mailbox->flagname[flag] = NULL;
1315 }
1316
1317 /* Read ACL */
1318 p = eol + 1;
1319 eol = memchr(p, '\n', sbuf.st_size - (p - base));
1320 if (!eol) {
1321 r = IMAP_MAILBOX_BADFORMAT;
1322 goto done;
1323 }
1324
1325 if (aclptr)
1326 *aclptr = xstrndup(p, eol-p);
1327
1328 done:
1329 if (base) map_free(&base, &len);
1330 return r;
1331 }
1332
1333 /* set a new ACL - only dirty if changed */
mailbox_set_acl(struct mailbox * mailbox,const char * acl)1334 EXPORTED int mailbox_set_acl(struct mailbox *mailbox, const char *acl)
1335 {
1336 if (!strcmpsafe(mailbox->acl, acl))
1337 return 0; /* no change */
1338 free(mailbox->acl);
1339 mailbox->acl = xstrdup(acl);
1340 mailbox->header_dirty = 1;
1341 return 0;
1342 }
1343
1344 /* set a new QUOTAROOT - only dirty if changed */
mailbox_set_quotaroot(struct mailbox * mailbox,const char * quotaroot)1345 EXPORTED int mailbox_set_quotaroot(struct mailbox *mailbox, const char *quotaroot)
1346 {
1347 if (mailbox->quotaroot) {
1348 if (quotaroot && !strcmp(mailbox->quotaroot, quotaroot))
1349 return 0; /* no change */
1350 free(mailbox->quotaroot);
1351 mailbox->quotaroot = NULL;
1352 }
1353 else {
1354 if (!quotaroot)
1355 return 0; /* no change */
1356 }
1357
1358 if (quotaroot)
1359 mailbox->quotaroot = xstrdup(quotaroot);
1360
1361 /* either way, it's changed, so dirty */
1362 mailbox->header_dirty = 1;
1363
1364 return 0;
1365 }
1366
1367 /* find or create a user flag - dirty header if change needed. If 'create'
1368 * is 1, then only 100 flags may be created. If >1, then you can use all 128 */
mailbox_user_flag(struct mailbox * mailbox,const char * flag,int * flagnum,int create)1369 EXPORTED int mailbox_user_flag(struct mailbox *mailbox, const char *flag,
1370 int *flagnum, int create)
1371 {
1372 int userflag;
1373 int emptyflag = -1;
1374
1375 for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
1376 if (mailbox->flagname[userflag]) {
1377 if (!strcasecmp(flag, mailbox->flagname[userflag]))
1378 break;
1379 }
1380 else if (emptyflag == -1) {
1381 emptyflag = userflag;
1382 }
1383 }
1384
1385 if (userflag == MAX_USER_FLAGS) {
1386 if (!create)
1387 return IMAP_NOTFOUND;
1388
1389 if (emptyflag == -1)
1390 return IMAP_USERFLAG_EXHAUSTED;
1391
1392 /* stop imapd exhausting flags */
1393 if (emptyflag >= 100 && create == 1) {
1394 syslog(LOG_ERR, "IOERROR: out of flags on %s (%s)",
1395 mailbox->name, flag);
1396 return IMAP_USERFLAG_EXHAUSTED;
1397 }
1398
1399 /* need to be index locked to make flag changes */
1400 if (!mailbox_index_islocked(mailbox, 1))
1401 return IMAP_MAILBOX_LOCKED;
1402
1403 if (!imparse_isatom(flag))
1404 return IMAP_INVALID_IDENTIFIER;
1405
1406 /* set the flag and mark the header dirty */
1407 userflag = emptyflag;
1408 mailbox->flagname[userflag] = xstrdup(flag);
1409 mailbox->header_dirty = 1;
1410 }
1411
1412 if (flagnum) *flagnum = userflag;
1413
1414 return 0;
1415 }
1416
1417 /* Remove a user flag from the mailbox, so that the slot can
1418 * be reused. Called from cyr_expire when we've made certain
1419 * that no record uses the flag anymore. */
mailbox_remove_user_flag(struct mailbox * mailbox,int flagnum)1420 EXPORTED int mailbox_remove_user_flag(struct mailbox *mailbox, int flagnum)
1421 {
1422 if (flagnum < 0 || flagnum >= MAX_USER_FLAGS)
1423 return IMAP_INTERNAL; /* invalid flag number */
1424
1425 if (!mailbox->flagname[flagnum])
1426 return 0; /* already gone */
1427
1428 /* need to be index locked to make flag changes */
1429 if (!mailbox_index_islocked(mailbox, 1))
1430 return IMAP_MAILBOX_LOCKED;
1431
1432 free(mailbox->flagname[flagnum]);
1433 mailbox->flagname[flagnum] = NULL;
1434 mailbox->header_dirty = 1;
1435 return 0;
1436 }
1437
mailbox_record_hasflag(struct mailbox * mailbox,const struct index_record * record,const char * flag)1438 EXPORTED int mailbox_record_hasflag(struct mailbox *mailbox,
1439 const struct index_record *record,
1440 const char *flag)
1441 {
1442 int userflag;
1443
1444 if (!mailbox) return 0;
1445 if (!flag) return 0;
1446 if (!record) return 0;
1447
1448 if (flag[0] == '\\') {
1449 if (!strcasecmp(flag, "\\answered"))
1450 return ((record->system_flags & FLAG_ANSWERED) ? 1 : 0);
1451 if (!strcasecmp(flag, "\\deleted"))
1452 return ((record->system_flags & FLAG_DELETED) ? 1 : 0);
1453 if (!strcasecmp(flag, "\\draft"))
1454 return ((record->system_flags & FLAG_DRAFT) ? 1 : 0);
1455 if (!strcasecmp(flag, "\\flagged"))
1456 return ((record->system_flags & FLAG_FLAGGED) ? 1 : 0);
1457 if (!strcasecmp(flag, "\\seen")) {
1458 /* NOTE: this is a special case because it depends
1459 * who the userid is. We will only return the user
1460 * or global seen value */
1461 return ((record->system_flags & FLAG_SEEN) ? 1 : 0);
1462 }
1463 /* unknown system flag is never present */
1464 return 0;
1465 }
1466
1467 if (mailbox_user_flag(mailbox, flag, &userflag, 0))
1468 return 0;
1469
1470 return ((record->user_flags[userflag/32] & (1<<(userflag&31))) ? 1 : 0);
1471 }
1472
mailbox_extract_flags(const struct mailbox * mailbox,const struct index_record * record,const char * userid)1473 EXPORTED strarray_t *mailbox_extract_flags(const struct mailbox *mailbox,
1474 const struct index_record *record,
1475 const char *userid)
1476 {
1477 int i;
1478 strarray_t *flags = strarray_new();
1479
1480 /* Note: we don't handle the external seen db here, on
1481 * the grounds that it would add complexity without
1482 * actually being useful to annotators */
1483 if (mailbox_internal_seen(mailbox, userid) && (record->system_flags & FLAG_SEEN))
1484 strarray_append(flags, "\\Seen");
1485
1486 if ((record->system_flags & FLAG_DELETED))
1487 strarray_append(flags, "\\Deleted");
1488 if ((record->system_flags & FLAG_DRAFT))
1489 strarray_append(flags, "\\Draft");
1490 if ((record->system_flags & FLAG_FLAGGED))
1491 strarray_append(flags, "\\Flagged");
1492 if ((record->system_flags & FLAG_ANSWERED))
1493 strarray_append(flags, "\\Answered");
1494
1495 for (i = 0 ; i < MAX_USER_FLAGS ; i++) {
1496 if (mailbox->flagname[i] &&
1497 (record->user_flags[i/32] & 1<<(i&31)))
1498 strarray_append(flags, mailbox->flagname[i]);
1499 }
1500
1501 return flags;
1502 }
1503
load_annot_cb(const char * mailbox,uint32_t uid,const char * entry,const char * userid,const struct buf * value,const struct annotate_metadata * mdata,void * rock)1504 static int load_annot_cb(const char *mailbox __attribute__((unused)),
1505 uint32_t uid __attribute__((unused)),
1506 const char *entry, const char *userid,
1507 const struct buf *value,
1508 const struct annotate_metadata *mdata __attribute__((unused)),
1509 void *rock)
1510 {
1511 struct entryattlist **eal = (struct entryattlist **)rock;
1512 const char *attrib = (userid[0] ? "value.priv" : "value.shared");
1513 setentryatt(eal, entry, attrib, value);
1514 return 0;
1515 }
1516
mailbox_extract_annots(const struct mailbox * mailbox,const struct index_record * record)1517 EXPORTED struct entryattlist *mailbox_extract_annots(const struct mailbox *mailbox,
1518 const struct index_record *record)
1519 {
1520 struct entryattlist *annots = NULL;
1521 int r = annotatemore_findall(mailbox->name, record->uid, "*", /*modseq*/0,
1522 load_annot_cb, &annots, /*flags*/0);
1523 if (r) return NULL;
1524 return annots;
1525 }
1526
mailbox_buf_to_index_header(const char * buf,size_t len,struct index_header * i)1527 static int mailbox_buf_to_index_header(const char *buf, size_t len,
1528 struct index_header *i)
1529 {
1530 uint32_t crc;
1531 bit32 qannot;
1532 size_t headerlen;
1533
1534 if (len < OFFSET_MINOR_VERSION+4)
1535 return IMAP_MAILBOX_BADFORMAT;
1536
1537 memset(i, 0, sizeof(struct index_header));
1538
1539 i->generation_no = ntohl(*((bit32 *)(buf+OFFSET_GENERATION_NO)));
1540 i->format = ntohl(*((bit32 *)(buf+OFFSET_FORMAT)));
1541 i->minor_version = ntohl(*((bit32 *)(buf+OFFSET_MINOR_VERSION)));
1542 switch (i->minor_version) {
1543 case 6:
1544 case 7:
1545 headerlen = 76;
1546 break;
1547 case 8:
1548 headerlen = 92;
1549 break;
1550 case 9:
1551 case 10:
1552 headerlen = 96;
1553 break;
1554 case 12:
1555 case 13:
1556 headerlen = 128;
1557 break;
1558 case 14:
1559 case 15:
1560 case 16:
1561 headerlen = 160;
1562 break;
1563 default:
1564 return IMAP_MAILBOX_BADFORMAT;
1565 }
1566 if (len < headerlen)
1567 return IMAP_MAILBOX_BADFORMAT;
1568
1569 i->start_offset = ntohl(*((bit32 *)(buf+OFFSET_START_OFFSET)));
1570 i->record_size = ntohl(*((bit32 *)(buf+OFFSET_RECORD_SIZE)));
1571 i->num_records = ntohl(*((bit32 *)(buf+OFFSET_NUM_RECORDS)));
1572 i->last_appenddate = ntohl(*((bit32 *)(buf+OFFSET_LAST_APPENDDATE)));
1573 i->last_uid = ntohl(*((bit32 *)(buf+OFFSET_LAST_UID)));
1574 i->quota_mailbox_used = align_ntohll(buf+OFFSET_QUOTA_MAILBOX_USED);
1575 i->pop3_last_login = ntohl(*((bit32 *)(buf+OFFSET_POP3_LAST_LOGIN)));
1576 i->uidvalidity = ntohl(*((bit32 *)(buf+OFFSET_UIDVALIDITY)));
1577 i->deleted = ntohl(*((bit32 *)(buf+OFFSET_DELETED)));
1578 i->answered = ntohl(*((bit32 *)(buf+OFFSET_ANSWERED)));
1579 i->flagged = ntohl(*((bit32 *)(buf+OFFSET_FLAGGED)));
1580 i->options = ntohl(*((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)));
1581 i->leaked_cache_records = ntohl(*((bit32 *)(buf+OFFSET_LEAKED_CACHE)));
1582 if (i->minor_version < 8) goto done;
1583
1584 i->highestmodseq = align_ntohll(buf+OFFSET_HIGHESTMODSEQ);
1585 if (i->minor_version < 12) goto done;
1586
1587 i->deletedmodseq = align_ntohll(buf+OFFSET_DELETEDMODSEQ);
1588 i->exists = ntohl(*((bit32 *)(buf+OFFSET_EXISTS)));
1589 i->first_expunged = ntohl(*((bit32 *)(buf+OFFSET_FIRST_EXPUNGED)));
1590 i->last_repack_time = ntohl(*((bit32 *)(buf+OFFSET_LAST_REPACK_TIME)));
1591 i->header_file_crc = ntohl(*((bit32 *)(buf+OFFSET_HEADER_FILE_CRC)));
1592 i->synccrcs.basic = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_BASIC)));
1593 i->recentuid = ntohl(*((bit32 *)(buf+OFFSET_RECENTUID)));
1594 i->recenttime = ntohl(*((bit32 *)(buf+OFFSET_RECENTTIME)));
1595
1596 if (i->minor_version < 13) goto crc;
1597
1598 i->pop3_show_after = ntohl(*((bit32 *)(buf+OFFSET_POP3_SHOW_AFTER)));
1599 qannot = ntohl(*((bit32 *)(buf+OFFSET_QUOTA_ANNOT_USED)));
1600 /* this field is stored as a 32b unsigned on disk but 64b signed
1601 * in memory, so we need to be careful about sign extension */
1602 i->quota_annot_used = (quota_t)((unsigned long long)qannot);
1603 i->synccrcs.annot = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_ANNOT)));
1604
1605 if (i->minor_version < 14) goto crc;
1606
1607 i->unseen = ntohl(*((bit32 *)(buf+OFFSET_UNSEEN)));
1608
1609 if (i->minor_version < 16) goto crc;
1610
1611 i->createdmodseq = align_ntohll(buf+OFFSET_MAILBOX_CREATEDMODSEQ);
1612
1613 crc:
1614 /* CRC is always the last 4 bytes */
1615 crc = ntohl(*((bit32 *)(buf+headerlen-4)));
1616 if (crc != crc32_map(buf, headerlen-4))
1617 return IMAP_MAILBOX_CHECKSUM;
1618
1619 done:
1620 if (!i->exists)
1621 i->options |= OPT_POP3_NEW_UIDL;
1622
1623 if (!i->highestmodseq)
1624 i->highestmodseq = 1;
1625
1626 if (i->minor_version < 12) {
1627 i->deletedmodseq = i->highestmodseq;
1628 i->exists = i->num_records;
1629 }
1630
1631 return 0;
1632 }
1633
mailbox_refresh_index_map(struct mailbox * mailbox)1634 static int mailbox_refresh_index_map(struct mailbox *mailbox)
1635 {
1636 size_t need_size;
1637 struct stat sbuf;
1638
1639 /* check if we need to extend the mmaped space for the index file
1640 * (i.e. new records appended since last read) */
1641 need_size = mailbox->i.start_offset +
1642 mailbox->i.num_records * mailbox->i.record_size;
1643 if (mailbox->index_size < need_size) {
1644 if (fstat(mailbox->index_fd, &sbuf) == -1)
1645 return IMAP_IOERROR;
1646
1647 if (sbuf.st_size < (int)need_size)
1648 return IMAP_MAILBOX_BADFORMAT;
1649
1650 mailbox->index_size = sbuf.st_size;
1651
1652 }
1653
1654 /* always refresh, we may be using map_nommap */
1655 map_refresh(mailbox->index_fd, 1, &mailbox->index_base,
1656 &mailbox->index_len, mailbox->index_size,
1657 "index", mailbox->name);
1658
1659 return 0;
1660 }
1661
mailbox_read_index_header(struct mailbox * mailbox)1662 static int mailbox_read_index_header(struct mailbox *mailbox)
1663 {
1664 int r;
1665
1666 /* no dirty mailboxes please */
1667 if (mailbox->i.dirty)
1668 abort();
1669
1670 /* need to be locked to ensure a consistent read - otherwise
1671 * a busy mailbox will get CRC errors due to rewrite happening
1672 * under our feet! */
1673 if (!mailbox_index_islocked(mailbox, 0))
1674 return IMAP_MAILBOX_LOCKED;
1675
1676 /* and of course it needs to exist and have at least enough
1677 * header to read the version number */
1678 if (!mailbox->index_base)
1679 return IMAP_MAILBOX_BADFORMAT;
1680
1681 /* need to make sure we're reading fresh data! */
1682 map_refresh(mailbox->index_fd, 1, &mailbox->index_base,
1683 &mailbox->index_len, mailbox->index_size,
1684 "index", mailbox->name);
1685
1686 r = mailbox_buf_to_index_header(mailbox->index_base, mailbox->index_len,
1687 &mailbox->i);
1688 if (r) return r;
1689
1690 r = mailbox_refresh_index_map(mailbox);
1691 if (r) return r;
1692
1693 return 0;
1694 }
1695
1696 /*
1697 * Read an index record from a mapped index file
1698 */
mailbox_buf_to_index_record(const char * buf,int version,struct index_record * record,int dirty)1699 static int mailbox_buf_to_index_record(const char *buf, int version,
1700 struct index_record *record, int dirty)
1701 {
1702 uint32_t crc;
1703 uint32_t stored_system_flags = 0;
1704 int n;
1705
1706 /* tracking fields - initialise */
1707 memset(record, 0, sizeof(struct index_record));
1708
1709 /* parse the shared bits first */
1710 record->uid = ntohl(*((bit32 *)(buf+OFFSET_UID)));
1711 record->internaldate = ntohl(*((bit32 *)(buf+OFFSET_INTERNALDATE)));
1712 record->sentdate = ntohl(*((bit32 *)(buf+OFFSET_SENTDATE)));
1713 record->size = ntohl(*((bit32 *)(buf+OFFSET_SIZE)));
1714 record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE)));
1715 record->gmtime = ntohl(*((bit32 *)(buf+OFFSET_GMTIME)));
1716 uint64_t cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET)));
1717 record->last_updated = ntohl(*((bit32 *)(buf+OFFSET_LAST_UPDATED)));
1718 stored_system_flags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS)));
1719
1720 /* de-serialise system flags and internal flags */
1721 record->system_flags = stored_system_flags & 0x000000ff;
1722 record->internal_flags = stored_system_flags & 0xff000000;
1723
1724 for (n = 0; n < MAX_USER_FLAGS/32; n++) {
1725 record->user_flags[n] = ntohl(*((bit32 *)(buf+OFFSET_USER_FLAGS+4*n)));
1726 }
1727 uint64_t cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION)));
1728
1729 /* keep the low bits of the version field for the version */
1730 record->cache_version = cache_version_field & 0xffff;
1731 /* use the high bits of the version field to extend the cache offset */
1732 record->cache_offset = cache_offset_field | ((cache_version_field & 0xffff0000) << 16);
1733
1734 if (version < 8)
1735 return 0;
1736
1737 if (version < 10) {
1738 /* modseq was at 72 before the GUID move */
1739 record->modseq = ntohll(*((bit64 *)(buf+72)));
1740 return 0;
1741 }
1742
1743 message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID);
1744 record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ)));
1745 if (version < 12)
1746 return 0;
1747
1748 /* THRID got inserted before cache_crc32 in version 12 */
1749 if (version < 13) {
1750 record->cache_crc = ntohl(*((bit32 *)(buf+88)));
1751
1752 if (dirty) return 0;
1753 /* check CRC32 */
1754 crc = crc32_map(buf, 92);
1755 if (crc != ntohl(*((bit32 *)(buf+92))))
1756 return IMAP_MAILBOX_CHECKSUM;
1757 return 0;
1758 }
1759
1760 record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID));
1761 if (version > 14) {
1762 record->savedate = ntohl(*((bit32 *)(buf+OFFSET_SAVEDATE)));
1763 }
1764
1765 /* createdmodseq was added in version 16, pushing the CRCs down */
1766 if (version < 16) {
1767 record->cache_crc = ntohl(*((bit32 *)(buf+96)));
1768
1769 if (dirty) return 0;
1770 /* check CRC32 */
1771 crc = crc32_map(buf, 100);
1772 if (crc != ntohl(*((bit32 *)(buf+100))))
1773 return IMAP_MAILBOX_CHECKSUM;
1774 return 0;
1775 }
1776
1777 record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ));
1778 record->cache_crc = ntohl(*((bit32 *)(buf+OFFSET_CACHE_CRC)));
1779
1780 if (dirty) return 0;
1781 /* check CRC32 */
1782 crc = crc32_map(buf, OFFSET_RECORD_CRC);
1783 if (crc != ntohl(*((bit32 *)(buf+OFFSET_RECORD_CRC))))
1784 return IMAP_MAILBOX_CHECKSUM;
1785
1786 return 0;
1787 }
1788
_find_change(struct mailbox * mailbox,uint32_t recno)1789 static struct index_change *_find_change(struct mailbox *mailbox, uint32_t recno)
1790 {
1791 uint32_t changeno = mailbox->index_change_map[recno % INDEX_MAP_SIZE];
1792
1793 while (changeno) {
1794 if (mailbox->index_changes[changeno-1].record.recno == recno)
1795 return &mailbox->index_changes[changeno-1];
1796 changeno = mailbox->index_changes[changeno-1].mapnext;
1797 }
1798
1799 return NULL;
1800 }
1801
_store_change(struct mailbox * mailbox,struct index_record * record,int flags)1802 static int _store_change(struct mailbox *mailbox, struct index_record *record, int flags)
1803 {
1804 struct index_change *change = _find_change(mailbox, record->recno);
1805
1806 if (!change) {
1807 mailbox->index_change_count++;
1808
1809 /* allocate a space if required */
1810 if (mailbox->index_change_count > mailbox->index_change_alloc) {
1811 mailbox->index_change_alloc += 256;
1812 mailbox->index_changes = xrealloc(mailbox->index_changes, sizeof(struct index_change) * mailbox->index_change_alloc);
1813 }
1814
1815 change = &mailbox->index_changes[mailbox->index_change_count-1];
1816 memset(change, 0, sizeof(struct index_change));
1817
1818 /* stitch into place */
1819 uint32_t pos = record->recno % INDEX_MAP_SIZE;
1820 change->mapnext = mailbox->index_change_map[pos];
1821 mailbox->index_change_map[pos] = mailbox->index_change_count; /* always non-zero */
1822 }
1823
1824 /* finally always copy the data into place */
1825 change->record = *record;
1826 change->flags = flags;
1827
1828 if (mailbox_cacherecord(mailbox, record)) {
1829 /* failed to load cache record */
1830 free(change->msgid);
1831 change->msgid = xstrdup("unknown");
1832 }
1833 else {
1834 char *c_env = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1,
1835 cacheitem_size(record, CACHE_ENVELOPE) - 2);
1836 char *envtokens[NUMENVTOKENS];
1837 parse_cached_envelope(c_env, envtokens, NUMENVTOKENS);
1838 free(change->msgid);
1839 change->msgid = xstrdup(envtokens[ENV_MSGID] ? envtokens[ENV_MSGID] : "unknown");
1840 free(c_env);
1841 }
1842
1843 annotate_state_t *astate = NULL;
1844 int r = mailbox_get_annotate_state(mailbox, record->uid, &astate);
1845 if (r) return r;
1846
1847 struct buf annotval = BUF_INITIALIZER;
1848 if (record->cid && record->basecid && record->basecid != record->cid)
1849 buf_printf(&annotval, "%016llx", record->basecid);
1850
1851 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval);
1852 buf_free(&annotval);
1853
1854 if (r) return r;
1855
1856 return 0;
1857 }
1858
_commit_one(struct mailbox * mailbox,struct index_change * change)1859 static int _commit_one(struct mailbox *mailbox, struct index_change *change)
1860 {
1861 indexbuffer_t ibuf;
1862 unsigned char *buf = ibuf.buf;
1863 size_t offset;
1864 struct index_record *record = &change->record;
1865 uint32_t recno = record->recno;
1866
1867 mailbox_index_record_to_buf(&change->record, mailbox->i.minor_version, buf);
1868
1869 offset = mailbox->i.start_offset + ((recno-1) * mailbox->i.record_size);
1870
1871 /* any failure here is a disaster! */
1872 if (lseek(mailbox->index_fd, offset, SEEK_SET) == -1) {
1873 syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
1874 recno, mailbox->name);
1875 return IMAP_IOERROR;
1876 }
1877
1878 if (retry_write(mailbox->index_fd, buf, mailbox->i.record_size) != mailbox->i.record_size) {
1879 syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
1880 recno, mailbox->name);
1881 return IMAP_IOERROR;
1882 }
1883
1884 /* audit logging */
1885 if (config_auditlog) {
1886 char flagstr[FLAGMAPSTR_MAXLEN];
1887 flags_to_str(record, flagstr);
1888 if (change->flags & CHANGE_ISAPPEND)
1889 /* note: messageid doesn't have <> wrappers because it already includes them */
1890 syslog(LOG_NOTICE, "auditlog: append sessionid=<%s> "
1891 "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> "
1892 "sysflags=<%s> guid=<%s> messageid=%s",
1893 session_id(), mailbox->name, mailbox->uniqueid, record->uid,
1894 record->modseq, flagstr,
1895 message_guid_encode(&record->guid), change->msgid);
1896
1897 if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(change->flags & CHANGE_WASEXPUNGED))
1898 syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> "
1899 "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> "
1900 "sysflags=<%s> guid=<%s>",
1901 session_id(), mailbox->name, mailbox->uniqueid, record->uid,
1902 record->modseq, flagstr,
1903 message_guid_encode(&record->guid));
1904
1905 if ((record->internal_flags & FLAG_INTERNAL_UNLINKED) && !(change->flags & CHANGE_WASUNLINKED))
1906 syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> "
1907 "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> "
1908 "sysflags=<%s> guid=<%s>",
1909 session_id(), mailbox->name, mailbox->uniqueid, record->uid,
1910 record->modseq, flagstr,
1911 message_guid_encode(&record->guid));
1912 }
1913
1914 return 0;
1915 }
1916
_cleanup_changes(struct mailbox * mailbox)1917 static void _cleanup_changes(struct mailbox *mailbox)
1918 {
1919 uint32_t i;
1920 for (i = 0; i < mailbox->index_change_count; i++) {
1921 free(mailbox->index_changes[i].msgid);
1922 }
1923 free(mailbox->index_changes);
1924 mailbox->index_changes = NULL;
1925 mailbox->index_change_count = 0;
1926 mailbox->index_change_alloc = 0;
1927 memset(mailbox->index_change_map, 0, sizeof(uint32_t)*INDEX_MAP_SIZE);
1928 }
1929
1930 /* qsort function for changes */
change_compar(const void * a,const void * b)1931 static int change_compar(const void *a, const void *b)
1932 {
1933 struct index_change *ac = (struct index_change *)a;
1934 struct index_change *bc = (struct index_change *)b;
1935
1936 if (ac->record.recno > bc->record.recno)
1937 return 1;
1938 if (ac->record.recno < bc->record.recno)
1939 return -1;
1940 return 0;
1941 }
1942
_commit_changes(struct mailbox * mailbox)1943 static int _commit_changes(struct mailbox *mailbox)
1944 {
1945 uint32_t i;
1946 int r;
1947
1948 if (!mailbox->index_change_count) return 0;
1949 mailbox->i.dirty = 1;
1950
1951 /* in which we throw away all our next pointers, but we don't care any more.
1952 * we just want to write in sensible order. Otherwise, there's no need to
1953 * do this sort at all */
1954 qsort(mailbox->index_changes, mailbox->index_change_count,
1955 sizeof(struct index_change), change_compar);
1956
1957 for (i = 0; i < mailbox->index_change_count; i++) {
1958 r = _commit_one(mailbox, &mailbox->index_changes[i]);
1959 if (r) return r; /* DAMN, we're screwed */
1960 }
1961
1962 _cleanup_changes(mailbox);
1963
1964 /* recalculate the size */
1965 mailbox->index_size = mailbox->i.start_offset + (mailbox->i.num_records * mailbox->i.record_size);
1966
1967 r = mailbox_refresh_index_map(mailbox);
1968 if (r) return r;
1969
1970 return 0;
1971 }
1972
mailbox_reload_index_record_dirty(struct mailbox * mailbox,struct index_record * record)1973 EXPORTED int mailbox_reload_index_record_dirty(struct mailbox *mailbox,
1974 struct index_record *record)
1975 {
1976 unsigned recno = record->recno;
1977 unsigned offset = mailbox->i.start_offset + (recno-1) * mailbox->i.record_size;
1978
1979 if (offset + mailbox->i.record_size > mailbox->index_size) {
1980 syslog(LOG_ERR,
1981 "IOERROR: index record %u for %s past end of file",
1982 recno, mailbox->name);
1983 return IMAP_IOERROR;
1984 }
1985
1986 const char *buf = mailbox->index_base + offset;
1987 mailbox_buf_to_index_record(buf, mailbox->i.minor_version, record, 1);
1988 record->recno = recno;
1989
1990 return 0;
1991 }
1992
1993
1994 /*
1995 * Read an index record from a mailbox
1996 */
mailbox_read_index_record(struct mailbox * mailbox,uint32_t recno,struct index_record * record)1997 static int mailbox_read_index_record(struct mailbox *mailbox,
1998 uint32_t recno,
1999 struct index_record *record)
2000 {
2001 const char *buf;
2002 unsigned offset;
2003 int r;
2004 struct index_change *change = _find_change(mailbox, recno);
2005
2006 if (change) {
2007 *record = change->record;
2008 return 0;
2009 }
2010
2011 offset = mailbox->i.start_offset + (recno-1) * mailbox->i.record_size;
2012
2013 if (offset + mailbox->i.record_size > mailbox->index_size) {
2014 syslog(LOG_ERR,
2015 "IOERROR: index record %u for %s past end of file",
2016 recno, mailbox->name);
2017 return IMAP_IOERROR;
2018 }
2019
2020 buf = mailbox->index_base + offset;
2021
2022 r = mailbox_buf_to_index_record(buf, mailbox->i.minor_version, record, 0);
2023
2024 record->recno = recno;
2025
2026 return r;
2027 }
2028
mailbox_read_basecid(struct mailbox * mailbox,const struct index_record * record)2029 EXPORTED int mailbox_read_basecid(struct mailbox *mailbox, const struct index_record *record)
2030 {
2031 if (record->basecid) return 0;
2032
2033 if (record->internal_flags & FLAG_INTERNAL_SPLITCONVERSATION) {
2034 struct buf annotval = BUF_INITIALIZER;
2035 mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "basethrid", "", &annotval);
2036 if (annotval.len == 16) {
2037 const char *p = buf_cstring(&annotval);
2038 /* we have a new canonical CID */
2039 struct index_record *backdoor = (struct index_record *)record;
2040 parsehex(p, &p, 16, &backdoor->basecid);
2041 }
2042 buf_free(&annotval);
2043 }
2044
2045 return 0;
2046 }
2047
mailbox_has_conversations(struct mailbox * mailbox)2048 EXPORTED int mailbox_has_conversations(struct mailbox *mailbox)
2049 {
2050 char *path;
2051
2052 /* not needed */
2053 if (!config_getswitch(IMAPOPT_CONVERSATIONS))
2054 return 0;
2055
2056 /* we never store data about deleted mailboxes */
2057 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
2058 return 0;
2059
2060 /* we never store data about submission mailboxes */
2061 if (mboxname_issubmissionmailbox(mailbox->name, mailbox->mbtype))
2062 return 0;
2063
2064 path = conversations_getmboxpath(mailbox->name);
2065 if (!path) return 0;
2066 free(path);
2067
2068 return 1;
2069 }
2070
mailbox_lock_conversations(struct mailbox * mailbox,int locktype)2071 static int mailbox_lock_conversations(struct mailbox *mailbox, int locktype)
2072 {
2073 /* does this mailbox have conversations? */
2074 if (!mailbox_has_conversations(mailbox))
2075 return 0;
2076
2077 /* already locked */
2078 struct conversations_state *cstate = conversations_get_mbox(mailbox->name);
2079 if (cstate) {
2080 if (locktype == LOCK_EXCLUSIVE) assert (!cstate->is_shared);
2081 return 0;
2082 }
2083
2084 if (locktype == LOCK_EXCLUSIVE) {
2085 return conversations_open_mbox(mailbox->name, 0/*shared*/, &mailbox->local_cstate);
2086 }
2087 else if (locktype == LOCK_SHARED) {
2088 return conversations_open_mbox(mailbox->name, 1/*shared*/, &mailbox->local_cstate);
2089 }
2090 else {
2091 /* this function does not support nonblocking locks */
2092 fatal("invalid locktype for conversations", EX_SOFTWARE);
2093 }
2094 }
2095
2096 #ifdef WITH_DAV
mailbox_open_caldav(struct mailbox * mailbox)2097 EXPORTED struct caldav_db *mailbox_open_caldav(struct mailbox *mailbox)
2098 {
2099 if (!mailbox->local_caldav) {
2100 mailbox->local_caldav = caldav_open_mailbox(mailbox);
2101 int r = caldav_begin(mailbox->local_caldav);
2102 if (r) {
2103 caldav_abort(mailbox->local_caldav);
2104 caldav_close(mailbox->local_caldav);
2105 mailbox->local_caldav = NULL;
2106 }
2107 }
2108 return mailbox->local_caldav;
2109 }
2110
mailbox_open_carddav(struct mailbox * mailbox)2111 EXPORTED struct carddav_db *mailbox_open_carddav(struct mailbox *mailbox)
2112 {
2113 if (!mailbox->local_carddav) {
2114 mailbox->local_carddav = carddav_open_mailbox(mailbox);
2115 int r = carddav_begin(mailbox->local_carddav);
2116 if (r) {
2117 carddav_abort(mailbox->local_carddav);
2118 carddav_close(mailbox->local_carddav);
2119 mailbox->local_carddav = NULL;
2120 }
2121 }
2122 return mailbox->local_carddav;
2123 }
2124
mailbox_open_webdav(struct mailbox * mailbox)2125 EXPORTED struct webdav_db *mailbox_open_webdav(struct mailbox *mailbox)
2126 {
2127 if (!mailbox->local_webdav) {
2128 mailbox->local_webdav = webdav_open_mailbox(mailbox);
2129 int r = webdav_begin(mailbox->local_webdav);
2130 if (r) {
2131 webdav_abort(mailbox->local_webdav);
2132 webdav_close(mailbox->local_webdav);
2133 mailbox->local_webdav = NULL;
2134 }
2135 }
2136 return mailbox->local_webdav;
2137 }
2138 #endif
2139
mailbox_getuid(struct mailbox * mailbox,uint32_t recno)2140 static uint32_t mailbox_getuid(struct mailbox *mailbox, uint32_t recno)
2141 {
2142 struct index_record record;
2143 record.uid = 0;
2144 /* XXX - cheaper memory-access reads? */
2145 mailbox_read_index_record(mailbox, recno, &record);
2146 return record.uid;
2147 }
2148
2149
2150 /*
2151 * Returns the recno of the message with UID 'uid'.
2152 * If no message with UID 'uid', returns the message with
2153 * the highest UID not greater than 'uid'.
2154 */
mailbox_finduid(struct mailbox * mailbox,uint32_t uid)2155 static uint32_t mailbox_finduid(struct mailbox *mailbox, uint32_t uid)
2156 {
2157 uint32_t low = 1;
2158 uint32_t high = mailbox->i.num_records;
2159 uint32_t mid;
2160 uint32_t miduid;
2161
2162 while (low <= high) {
2163 mid = (high - low)/2 + low;
2164 miduid = mailbox_getuid(mailbox, mid);
2165 if (miduid == uid)
2166 return mid;
2167 else if (miduid > uid)
2168 high = mid - 1;
2169 else
2170 low = mid + 1;
2171 }
2172 return high;
2173 }
2174
2175 /*
2176 * Perform a binary search on the mailbox index file to read the record
2177 * for uid 'uid' into 'record'. If 'oldrecord' is not NULL then it is
2178 * assumed to point a correct and current index record from an earlier
2179 * call, and the search is bounded by that record. Returns 0 on success
2180 * or an IMAP error code on failure.
2181 */
mailbox_find_index_record(struct mailbox * mailbox,uint32_t uid,struct index_record * record)2182 EXPORTED int mailbox_find_index_record(struct mailbox *mailbox, uint32_t uid,
2183 struct index_record *record)
2184 {
2185 uint32_t recno = mailbox_finduid(mailbox, uid);
2186 /* no records? */
2187 if (!recno) return IMAP_NOTFOUND;
2188
2189 int r = mailbox_read_index_record(mailbox, recno, record);
2190 /* failed read? */
2191 if (r) return r;
2192
2193 /* wasn't the actual record? */
2194 if (record->uid != uid) return IMAP_NOTFOUND;
2195
2196 return 0;
2197 }
2198
2199 /*
2200 * Lock the index file for 'mailbox'. Reread index file header if necessary.
2201 */
mailbox_lock_index_internal(struct mailbox * mailbox,int locktype)2202 static int mailbox_lock_index_internal(struct mailbox *mailbox, int locktype)
2203 {
2204 struct stat sbuf;
2205 int r = 0;
2206 const char *header_fname = mailbox_meta_fname(mailbox, META_HEADER);
2207 const char *index_fname = mailbox_meta_fname(mailbox, META_INDEX);
2208
2209 assert(mailbox->index_fd != -1);
2210 assert(!mailbox->index_locktype);
2211
2212 r = 0;
2213
2214 if (locktype == LOCK_EXCLUSIVE) {
2215 /* handle read-only case cleanly - we need to re-open read-write first! */
2216 if (mailbox->is_readonly) {
2217 mailbox->is_readonly = 0;
2218 r = mailbox_open_index(mailbox);
2219 }
2220 if (!r) r = lock_blocking(mailbox->index_fd, index_fname);
2221 }
2222 else if (locktype == LOCK_SHARED) {
2223 r = lock_shared(mailbox->index_fd, index_fname);
2224 }
2225 else {
2226 /* this function does not support nonblocking locks */
2227 fatal("invalid locktype for index", EX_SOFTWARE);
2228 }
2229
2230 /* double check that the index exists and has at least enough
2231 * data to check the version number */
2232 if (!r) {
2233 if (!mailbox->index_base)
2234 r = IMAP_MAILBOX_BADFORMAT;
2235 else if (mailbox->index_size < OFFSET_NUM_RECORDS)
2236 r = IMAP_MAILBOX_BADFORMAT;
2237 if (r)
2238 lock_unlock(mailbox->index_fd, index_fname);
2239 }
2240
2241 if (r) {
2242 syslog(LOG_ERR, "IOERROR: locking index for %s: %s",
2243 mailbox->name, error_message(r));
2244 return IMAP_IOERROR;
2245 }
2246
2247 mailbox->index_locktype = locktype;
2248 gettimeofday(&mailbox->starttime, 0);
2249
2250 r = stat(header_fname, &sbuf);
2251 if (r == -1) {
2252 syslog(LOG_ERR, "IOERROR: stating header %s for %s: %m",
2253 header_fname, mailbox->name);
2254 mailbox_unlock_index(mailbox, NULL);
2255 return IMAP_IOERROR;
2256 }
2257
2258 /* has the header file changed? */
2259 if (sbuf.st_ino != mailbox->header_file_ino) {
2260 r = mailbox_read_header(mailbox, NULL);
2261 if (r) {
2262 syslog(LOG_ERR, "IOERROR: reading header for %s: %s",
2263 mailbox->name, error_message(r));
2264 mailbox_unlock_index(mailbox, NULL);
2265 return r;
2266 }
2267 }
2268
2269 /* release caches */
2270 int i;
2271 for (i = 0; i < mailbox->caches.count; i++) {
2272 struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i);
2273 mappedfile_close(&cachefile);
2274 }
2275 ptrarray_fini(&mailbox->caches);
2276
2277 /* note: it's guaranteed by our outer cyrus.lock lock that the
2278 * cyrus.index and cyrus.cache files are never rewritten, so
2279 * we're safe to just extend the map if needed */
2280 r = mailbox_read_index_header(mailbox);
2281 if (r) {
2282 syslog(LOG_ERR, "IOERROR: refreshing index for %s: %s",
2283 mailbox->name, error_message(r));
2284 mailbox_unlock_index(mailbox, NULL);
2285 return r;
2286 }
2287
2288 /* check the CRC */
2289 if (mailbox->header_file_crc && mailbox->i.header_file_crc &&
2290 mailbox->header_file_crc != mailbox->i.header_file_crc) {
2291 syslog(LOG_WARNING, "Header CRC mismatch for mailbox %s: %08X %08X",
2292 mailbox->name, (unsigned int)mailbox->header_file_crc,
2293 (unsigned int)mailbox->i.header_file_crc);
2294 }
2295
2296 return 0;
2297 }
2298
mailbox_lock_index(struct mailbox * mailbox,int locktype)2299 EXPORTED int mailbox_lock_index(struct mailbox *mailbox, int locktype)
2300 {
2301 int r = 0;
2302
2303 /* always lock the conversations DB, since if we have the index file
2304 * locked at all, we can't open it later */
2305 r = mailbox_lock_conversations(mailbox, locktype);
2306 if (r) return r;
2307
2308 r = mailbox_lock_index_internal(mailbox, locktype);
2309 if (r) return r;
2310
2311 /* otherwise, sanity checks for regular use, but not for internal
2312 * use during cleanup */
2313
2314 /* we may be in the process of deleting this mailbox */
2315 if (mailbox->i.options & OPT_MAILBOX_DELETED) {
2316 mailbox_unlock_index(mailbox, NULL);
2317 return IMAP_MAILBOX_NONEXISTENT;
2318 }
2319
2320 return 0;
2321 }
2322
2323 /*
2324 * Release lock on the index file for 'mailbox'
2325 */
mailbox_unlock_index(struct mailbox * mailbox,struct statusdata * sdata)2326 EXPORTED void mailbox_unlock_index(struct mailbox *mailbox, struct statusdata *sdata)
2327 {
2328 struct statusdata mysdata = STATUSDATA_INIT;
2329 struct timeval endtime;
2330 double timediff;
2331 int r;
2332 const char *index_fname = mailbox_meta_fname(mailbox, META_INDEX);
2333
2334 /* this is kinda awful, but too much code expects it to work, and the
2335 * refcounting isn't good about partial commit/abort and all the
2336 * unwinding, so here you are. At least if you mailbox_abort, then
2337 * it resets the dirty flags, so this becomes a NOOP during close */
2338 /* naughty - you can't unlock a dirty mailbox! */
2339 r = mailbox_commit(mailbox);
2340 if (r) {
2341 syslog(LOG_ERR, "IOERROR: failed to commit mailbox %s, "
2342 "probably need to reconstruct",
2343 mailbox->name);
2344 abort();
2345 }
2346
2347 if (mailbox->has_changed) {
2348 if (updatenotifier) updatenotifier(mailbox->name);
2349 sync_log_mailbox(mailbox->name);
2350
2351 if (!sdata) {
2352 status_fill_mailbox(mailbox, &mysdata);
2353 sdata = &mysdata;
2354 }
2355
2356 mailbox->has_changed = 0;
2357 }
2358
2359 // we always write if given new statusdata, or if we changed the mailbox
2360 if (sdata)
2361 statuscache_invalidate(mailbox->name, sdata);
2362
2363 if (mailbox->index_locktype) {
2364 if (lock_unlock(mailbox->index_fd, index_fname))
2365 syslog(LOG_ERR, "IOERROR: unlocking index of %s: %m",
2366 mailbox->name);
2367 mailbox->index_locktype = 0;
2368 }
2369
2370 gettimeofday(&endtime, 0);
2371 timediff = timesub(&mailbox->starttime, &endtime);
2372 if (timediff > 1.0) {
2373 syslog(LOG_NOTICE, "mailbox: longlock %s for %0.1f seconds",
2374 mailbox->name, timediff);
2375 }
2376
2377 if (mailbox->local_cstate) {
2378 int r = conversations_commit(&mailbox->local_cstate);
2379 if (r)
2380 syslog(LOG_ERR, "Error committing to conversations database for mailbox %s: %s",
2381 mailbox->name, error_message(r));
2382 }
2383 }
2384
2385 /*
2386 * Write the header file for 'mailbox'
2387 */
mailbox_commit_header(struct mailbox * mailbox)2388 static int mailbox_commit_header(struct mailbox *mailbox)
2389 {
2390 int flag;
2391 int fd;
2392 int r = 0;
2393 const char *quotaroot;
2394 const char *newfname;
2395 struct iovec iov[10];
2396 int niov;
2397
2398 if (!mailbox->header_dirty)
2399 return 0; /* nothing to write! */
2400
2401 /* we actually do all header actions under an INDEX lock, because
2402 * we need to write the crc32 to be consistent! */
2403 assert(mailbox_index_islocked(mailbox, 1));
2404
2405 newfname = mailbox_meta_newfname(mailbox, META_HEADER);
2406
2407 fd = open(newfname, O_CREAT | O_TRUNC | O_RDWR, 0666);
2408 if (fd == -1) {
2409 syslog(LOG_ERR, "IOERROR: opening %s: %m", newfname);
2410 return IMAP_IOERROR;
2411 }
2412
2413 /* Write magic header, do NOT write the trailing NUL */
2414 r = write(fd, MAILBOX_HEADER_MAGIC,
2415 sizeof(MAILBOX_HEADER_MAGIC) - 1);
2416
2417 if (r != -1) {
2418 niov = 0;
2419 quotaroot = mailbox->quotaroot ? mailbox->quotaroot : "";
2420 WRITEV_ADDSTR_TO_IOVEC(iov,niov,quotaroot);
2421 WRITEV_ADD_TO_IOVEC(iov,niov,"\t",1);
2422 WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->uniqueid);
2423 WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
2424 r = retry_writev(fd, iov, niov);
2425 }
2426
2427 if (r != -1) {
2428 for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
2429 if (mailbox->flagname[flag]) {
2430 niov = 0;
2431 WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->flagname[flag]);
2432 WRITEV_ADD_TO_IOVEC(iov,niov," ",1);
2433 r = retry_writev(fd, iov, niov);
2434 if(r == -1) break;
2435 }
2436 }
2437 }
2438
2439 if (r != -1) {
2440 niov = 0;
2441 WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
2442 WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->acl);
2443 WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
2444 r = retry_writev(fd, iov, niov);
2445 }
2446
2447 if (r == -1 || fsync(fd)) {
2448 syslog(LOG_ERR, "IOERROR: writing %s: %m", newfname);
2449 close(fd);
2450 unlink(newfname);
2451 return IMAP_IOERROR;
2452 }
2453
2454 close(fd);
2455
2456 /* rename the new header file over the old one */
2457 r = mailbox_meta_rename(mailbox, META_HEADER);
2458 if (r) return r;
2459 mailbox->header_dirty = 0; /* we wrote it out, so not dirty any more */
2460
2461 /* re-read the header */
2462 r = mailbox_read_header(mailbox, NULL);
2463 if (r) return r;
2464
2465 /* copy the new CRC into the index header */
2466 mailbox->i.header_file_crc = mailbox->header_file_crc;
2467 mailbox_index_dirty(mailbox);
2468
2469 return 0;
2470 }
2471
mailbox_index_header_to_buf(struct index_header * i,unsigned char * buf)2472 static bit32 mailbox_index_header_to_buf(struct index_header *i, unsigned char *buf)
2473 {
2474 bit32 crc;
2475 bit32 options = i->options & MAILBOX_OPT_VALID;
2476 size_t headerlen = INDEX_HEADER_SIZE;
2477
2478 memset(buf, 0, INDEX_HEADER_SIZE); /* buffer is always this big, and aligned */
2479
2480 assert (i->minor_version >= 6);
2481
2482 *((bit32 *)(buf+OFFSET_GENERATION_NO)) = htonl(i->generation_no);
2483 *((bit32 *)(buf+OFFSET_FORMAT)) = htonl(i->format);
2484 *((bit32 *)(buf+OFFSET_MINOR_VERSION)) = htonl(i->minor_version);
2485 *((bit32 *)(buf+OFFSET_START_OFFSET)) = htonl(i->start_offset);
2486 *((bit32 *)(buf+OFFSET_RECORD_SIZE)) = htonl(i->record_size);
2487 /* this was moved to make upgrades clean, because num_records was
2488 * the same as exists back then, we didn't keep expunged in the
2489 * record - but we always have to write NUM_RECORDS so that expunged
2490 * handing over repack works */
2491 *((bit32 *)(buf+OFFSET_NUM_RECORDS)) = htonl(i->num_records);
2492 *((bit32 *)(buf+OFFSET_LAST_APPENDDATE)) = htonl(i->last_appenddate);
2493 *((bit32 *)(buf+OFFSET_LAST_UID)) = htonl(i->last_uid);
2494
2495 /* quotas may be 64bit now */
2496 align_htonll(buf+OFFSET_QUOTA_MAILBOX_USED, i->quota_mailbox_used);
2497
2498 *((bit32 *)(buf+OFFSET_POP3_LAST_LOGIN)) = htonl(i->pop3_last_login);
2499 *((bit32 *)(buf+OFFSET_UIDVALIDITY)) = htonl(i->uidvalidity);
2500 *((bit32 *)(buf+OFFSET_DELETED)) = htonl(i->deleted);
2501 *((bit32 *)(buf+OFFSET_ANSWERED)) = htonl(i->answered);
2502 *((bit32 *)(buf+OFFSET_FLAGGED)) = htonl(i->flagged);
2503 if (i->minor_version < 8) {
2504 /* this was called OFFSET_POP3_NEW_UIDL and was only zero or one */
2505 *((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(options&1);
2506 return 0; /* no CRC32 support */
2507 }
2508
2509 /* otherwise we have options and modseqs */
2510 *((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(options);
2511 *((bit32 *)(buf+OFFSET_LEAKED_CACHE)) = htonl(i->leaked_cache_records);
2512 align_htonll(buf+OFFSET_HIGHESTMODSEQ, i->highestmodseq);
2513
2514 /* and that's where it stopped until version 2.4.0 with index version 12 (ignoring
2515 * version 11, which doesn't exist in the wild */
2516 if (i->minor_version < 12) {
2517 return 0;
2518 }
2519
2520 align_htonll(buf+OFFSET_DELETEDMODSEQ, i->deletedmodseq);
2521 *((bit32 *)(buf+OFFSET_EXISTS)) = htonl(i->exists);
2522 *((bit32 *)(buf+OFFSET_FIRST_EXPUNGED)) = htonl(i->first_expunged);
2523 *((bit32 *)(buf+OFFSET_LAST_REPACK_TIME)) = htonl(i->last_repack_time);
2524 *((bit32 *)(buf+OFFSET_HEADER_FILE_CRC)) = htonl(i->header_file_crc);
2525 *((bit32 *)(buf+OFFSET_SYNCCRCS_BASIC)) = htonl(i->synccrcs.basic);
2526 *((bit32 *)(buf+OFFSET_RECENTUID)) = htonl(i->recentuid);
2527 *((bit32 *)(buf+OFFSET_RECENTTIME)) = htonl(i->recenttime);
2528 if (i->minor_version > 12) {
2529 /* these were added in version 13, but replaced zero-byte fields in
2530 * in version 12, so if we don't write them then the CRC will still
2531 * be correct for version 12, since the header size didn't change */
2532 *((bit32 *)(buf+OFFSET_POP3_SHOW_AFTER)) = htonl(i->pop3_show_after);
2533 /* this field is 64b in memory but 32b on disk - as it counts
2534 * bytes stored in dbs and the dbs are 32b anyway there should
2535 * be no problem */
2536 *((bit32 *)(buf+OFFSET_QUOTA_ANNOT_USED)) = htonl((bit32)i->quota_annot_used);
2537 *((bit32 *)(buf+OFFSET_SYNCCRCS_ANNOT)) = htonl(i->synccrcs.annot);
2538 }
2539
2540 if (i->minor_version > 13) {
2541 *((bit32 *)(buf+OFFSET_UNSEEN)) = htonl(i->unseen);
2542 }
2543 else {
2544 /* we expanded the file size in version 14 */
2545 headerlen = 128;
2546 }
2547
2548 if (i->minor_version > 15) {
2549 align_htonll(buf+OFFSET_MAILBOX_CREATEDMODSEQ, i->createdmodseq);
2550 }
2551
2552 /* Update checksum */
2553 crc = htonl(crc32_map((char *)buf, headerlen-4));
2554 *((bit32 *)(buf+headerlen-4)) = crc;
2555
2556 return crc;
2557 }
2558
mailbox_commit_quota(struct mailbox * mailbox)2559 HIDDEN int mailbox_commit_quota(struct mailbox *mailbox)
2560 {
2561 int res;
2562 int changed = 0;
2563 quota_t quota_usage[QUOTA_NUMRESOURCES];
2564
2565 /* not dirty */
2566 if (!mailbox->quota_dirty)
2567 return 0;
2568
2569 mailbox->quota_dirty = 0;
2570
2571 /* no quota root means we don't track quota. That's OK */
2572 if (!mailbox->quotaroot)
2573 return 0;
2574
2575 mailbox_get_usage(mailbox, quota_usage);
2576 for (res = 0; res < QUOTA_NUMRESOURCES; res++) {
2577 quota_usage[res] -= mailbox->quota_previously_used[res];
2578 if (quota_usage[res] != 0) {
2579 changed++;
2580 }
2581 }
2582 /* unchanged */
2583 if (!changed)
2584 return 0;
2585
2586 assert(mailbox_index_islocked(mailbox, 1));
2587
2588 quota_update_useds(mailbox->quotaroot, quota_usage,
2589 mailbox->name, mailbox->silentchanges);
2590 /* XXX - fail upon issue? It's tempting */
2591
2592 return 0;
2593 }
2594
2595
2596 /*
2597 * Abort the changes to a mailbox
2598 */
mailbox_abort(struct mailbox * mailbox)2599 EXPORTED int mailbox_abort(struct mailbox *mailbox)
2600 {
2601 int r;
2602
2603 #ifdef WITH_DAV
2604 r = mailbox_abort_dav(mailbox);
2605 if (r) return r;
2606 #endif
2607
2608 /* try to commit sub parts first */
2609 r = mailbox_abort_cache(mailbox);
2610 if (r) return r;
2611
2612 annotate_state_abort(&mailbox->annot_state);
2613
2614 if (!mailbox->i.dirty)
2615 return 0;
2616
2617 assert(mailbox_index_islocked(mailbox, 1));
2618
2619 /* remove all dirty flags! */
2620 mailbox->i.dirty = 0;
2621 mailbox->modseq_dirty = 0;
2622 mailbox->header_dirty = 0;
2623
2624 /* removed cached changes */
2625 _cleanup_changes(mailbox);
2626
2627 /* we re-read the header and index header to wipe
2628 * away all the changed values */
2629 r = mailbox_read_header(mailbox, NULL);
2630 if (r) return r;
2631
2632 r = mailbox_read_index_header(mailbox);
2633 if (r) return r;
2634
2635 return 0;
2636 }
2637
2638 /*
2639 * Write the index header for 'mailbox'
2640 */
mailbox_commit(struct mailbox * mailbox)2641 EXPORTED int mailbox_commit(struct mailbox *mailbox)
2642 {
2643 /* XXX - ibuf for alignment? */
2644 static unsigned char buf[INDEX_HEADER_SIZE];
2645 int n, r;
2646
2647 /* try to commit sub parts first */
2648 #ifdef WITH_DAV
2649 r = mailbox_commit_dav(mailbox);
2650 if (r) return r;
2651 #endif
2652
2653 r = mailbox_commit_cache(mailbox);
2654 if (r) return r;
2655
2656 r = mailbox_commit_quota(mailbox);
2657 if (r) return r;
2658
2659 r = annotate_state_commit(&mailbox->annot_state);
2660 if (r) return r;
2661
2662 r = mailbox_commit_header(mailbox);
2663 if (r) return r;
2664
2665 if (!mailbox->i.dirty)
2666 return 0;
2667
2668 mboxname_setmodseq(mailbox->name,
2669 mailbox->i.highestmodseq,
2670 mailbox->mbtype, /*dofolder*/0);
2671
2672 assert(mailbox_index_islocked(mailbox, 1));
2673
2674 r = _commit_changes(mailbox);
2675 if (r) return r;
2676
2677 /* always update xconvmodseq, it might have been done by annotations */
2678 r = mailbox_update_xconvmodseq(mailbox, mailbox->i.highestmodseq, /*force*/0);
2679 if (r) return r;
2680
2681 mailbox_index_header_to_buf(&mailbox->i, buf);
2682
2683 lseek(mailbox->index_fd, 0, SEEK_SET);
2684 n = retry_write(mailbox->index_fd, buf, mailbox->i.start_offset);
2685 if (n < 0 || fsync(mailbox->index_fd)) {
2686 syslog(LOG_ERR, "IOERROR: writing index header for %s: %m",
2687 mailbox->name);
2688 return IMAP_IOERROR;
2689 }
2690
2691 if (config_auditlog && mailbox->modseq_dirty)
2692 syslog(LOG_NOTICE, "auditlog: modseq sessionid=<%s> "
2693 "mailbox=<%s> uniqueid=<%s> highestmodseq=<" MODSEQ_FMT ">",
2694 session_id(), mailbox->name, mailbox->uniqueid,
2695 mailbox->i.highestmodseq);
2696
2697 if (mailbox->modseq_dirty) {
2698 struct mboxevent *mboxevent = mboxevent_new(EVENT_MAILBOX_MODSEQ);
2699 mboxevent_extract_mailbox(mboxevent, mailbox);
2700 mboxevent_set_access(mboxevent, NULL, NULL, "", mailbox->name, 0);
2701 mboxevent_notify(&mboxevent);
2702 mboxevent_free(&mboxevent);
2703 }
2704
2705 /* remove all dirty flags! */
2706 mailbox->i.dirty = 0;
2707 mailbox->modseq_dirty = 0;
2708 mailbox->header_dirty = 0;
2709
2710 /* label changes for later logging */
2711 mailbox->has_changed = 1;
2712
2713 return 0;
2714 }
2715
2716 /*
2717 * Put an index record into a buffer suitable for writing to a file.
2718 */
mailbox_index_record_to_buf(struct index_record * record,int version,unsigned char * buf)2719 static bit32 mailbox_index_record_to_buf(struct index_record *record, int version,
2720 unsigned char *buf)
2721 {
2722 int n;
2723 bit32 crc;
2724 uint32_t system_flags = 0;
2725
2726 memset(buf, 0, INDEX_RECORD_SIZE);
2727
2728 /* keep the low bits of the offset in the offset field */
2729 uint32_t cache_offset_field = record->cache_offset & 0xffffffff;
2730 /* mix in the high bits of the offset to the top half of the version field */
2731 uint32_t cache_version_field = (uint32_t)record->cache_version | (record->cache_offset & 0xffff00000000) >> 16;
2732
2733 *((bit32 *)(buf+OFFSET_UID)) = htonl(record->uid);
2734 *((bit32 *)(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate);
2735 *((bit32 *)(buf+OFFSET_SENTDATE)) = htonl(record->sentdate);
2736 *((bit32 *)(buf+OFFSET_SIZE)) = htonl(record->size);
2737 *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size);
2738 if (version >= 12) {
2739 *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->gmtime);
2740 }
2741 else {
2742 /* content_offset was always the same */
2743 *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->header_size);
2744 }
2745 *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field);
2746 *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(record->last_updated);
2747
2748 /* serialise system flags and internal flags */
2749 system_flags = record->system_flags | record->internal_flags;
2750 *((bit32 *)(buf+OFFSET_SYSTEM_FLAGS)) = htonl(system_flags);
2751
2752 for (n = 0; n < MAX_USER_FLAGS/32; n++) {
2753 *((bit32 *)(buf+OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]);
2754 }
2755 if (version > 14) {
2756 *((bit32 *)(buf+OFFSET_SAVEDATE)) = htonl(record->savedate);
2757 }
2758 else {
2759 *((bit32 *)(buf+OFFSET_SAVEDATE)) = 0; // blank out old content_lines
2760 }
2761 *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field);
2762
2763 /* versions less than 8 had no modseq */
2764 if (version < 8) {
2765 return 0;
2766 }
2767
2768 /* versions 8 and 9 only had a smaller UUID, which we will ignore,
2769 * but the modseq existed and was at offset 72 and 76 */
2770 if (version < 10) {
2771 *((bit64 *)(buf+72)) = htonll(record->modseq);
2772 return 0;
2773 }
2774
2775 /* otherwise we have the GUID and MODSEQ in their current place */
2776 message_guid_export(&record->guid, (char *)buf+OFFSET_MESSAGE_GUID);
2777 *((bit64 *)(buf+OFFSET_MODSEQ)) = htonll(record->modseq);
2778
2779 /* version 12 added the CACHE_CRC and RECORD_CRC, but at a lower point */
2780 if (version < 13) {
2781 *((bit32 *)(buf+88)) = htonl(record->cache_crc);
2782 /* calculate the checksum */
2783 crc = crc32_map((char *)buf, 92);
2784 *((bit32 *)(buf+92)) = htonl(crc);
2785 return crc;
2786 }
2787
2788 *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid);
2789
2790 /* version 16 added createdmodseq, pushing the CRCs down */
2791 if (version < 16) {
2792 *((bit32 *)(buf+96)) = htonl(record->cache_crc);
2793 crc = crc32_map((char *)buf, 100);
2794 *((bit32 *)(buf+100)) = htonl(crc);
2795 return crc;
2796 }
2797
2798 *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq);
2799 *((bit32 *)(buf+OFFSET_CACHE_CRC)) = htonl(record->cache_crc);
2800
2801 /* calculate the checksum */
2802 crc = crc32_map((char *)buf, OFFSET_RECORD_CRC);
2803 *((bit32 *)(buf+OFFSET_RECORD_CRC)) = htonl(crc);
2804
2805 return crc;
2806 }
2807
2808
mailbox_quota_dirty(struct mailbox * mailbox)2809 static void mailbox_quota_dirty(struct mailbox *mailbox)
2810 {
2811 /* track quota use */
2812 if (!mailbox->quota_dirty) {
2813 mailbox->quota_dirty = 1;
2814 mailbox_get_usage(mailbox, mailbox->quota_previously_used);
2815 }
2816 }
2817
header_update_counts(struct index_header * i,const struct index_record * record,int is_add)2818 static void header_update_counts(struct index_header *i,
2819 const struct index_record *record,
2820 int is_add)
2821 {
2822 int num = is_add ? 1 : -1;
2823
2824 /* we don't track counts for EXPUNGED records */
2825 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED)
2826 return;
2827
2828 /* update mailbox header fields */
2829 if (record->system_flags & FLAG_ANSWERED)
2830 i->answered += num;
2831
2832 if (record->system_flags & FLAG_FLAGGED)
2833 i->flagged += num;
2834
2835 if (record->system_flags & FLAG_DELETED)
2836 i->deleted += num;
2837
2838 if (!(record->system_flags & FLAG_SEEN))
2839 i->unseen += num;
2840
2841 if (is_add) {
2842 i->exists++;
2843 i->quota_mailbox_used += record->size;
2844 }
2845 else {
2846 if (i->exists) i->exists--;
2847
2848 /* corruption prevention - check we don't go negative */
2849 if (i->quota_mailbox_used > record->size)
2850 i->quota_mailbox_used -= record->size;
2851 else
2852 i->quota_mailbox_used = 0;
2853 }
2854 }
2855
2856 /*************************** Sync CRC ***************************/
2857
2858 struct annot_calc_rock
2859 {
2860 struct mailbox *mailbox;
2861 uint32_t annot;
2862 quota_t used;
2863 };
2864
2865 /* this is the algorithm from version 2.4, it's locked in */
crc_basic(const struct mailbox * mailbox,const struct index_record * record)2866 static uint32_t crc_basic(const struct mailbox *mailbox,
2867 const struct index_record *record)
2868 {
2869 char buf[4096];
2870 uint32_t flagcrc = 0;
2871 int flag;
2872
2873 /* expunged flags have no sync CRC */
2874 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED)
2875 return 0;
2876
2877 /* calculate an XORed CRC32 over all the flags on the message, so no
2878 * matter what order they are store in the header, the final value
2879 * is the same */
2880 if (record->system_flags & FLAG_DELETED)
2881 flagcrc ^= crc32_cstring("\\deleted");
2882 if (record->system_flags & FLAG_ANSWERED)
2883 flagcrc ^= crc32_cstring("\\answered");
2884 if (record->system_flags & FLAG_FLAGGED)
2885 flagcrc ^= crc32_cstring("\\flagged");
2886 if (record->system_flags & FLAG_DRAFT)
2887 flagcrc ^= crc32_cstring("\\draft");
2888 if (record->system_flags & FLAG_SEEN)
2889 flagcrc ^= crc32_cstring("\\seen");
2890
2891 for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
2892 if (!mailbox->flagname[flag])
2893 continue;
2894 if (!(record->user_flags[flag/32] & (1<<(flag&31))))
2895 continue;
2896 /* need to compare without case being significant */
2897 strlcpy(buf, mailbox->flagname[flag], 4096);
2898 lcase(buf);
2899 flagcrc ^= crc32_cstring(buf);
2900 }
2901
2902 snprintf(buf, sizeof(buf), "%u " MODSEQ_FMT " " TIME_T_FMT " (%u) " TIME_T_FMT " %s",
2903 record->uid, record->modseq, record->last_updated,
2904 flagcrc,
2905 record->internaldate,
2906 message_guid_encode(&record->guid));
2907
2908 return crc32_cstring(buf);
2909 }
2910
crc_annot(unsigned int uid,const char * entry,const char * userid,const struct buf * value)2911 static uint32_t crc_annot(unsigned int uid, const char *entry,
2912 const char *userid, const struct buf *value)
2913 {
2914 struct buf buf = BUF_INITIALIZER;
2915 uint32_t res = 0;
2916
2917 // ignore everything with a NULL userid, it's bogus!
2918 if (!userid) return 0;
2919
2920 buf_printf(&buf, "%u %s %s ", uid, entry, userid);
2921 buf_append(&buf, value);
2922 res = crc32_buf(&buf);
2923 buf_free(&buf);
2924
2925 return res;
2926 }
2927
mailbox_is_virtannot(struct mailbox * mailbox,const char * entry)2928 static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry)
2929 {
2930 if (mailbox->i.minor_version < 13) return 0;
2931 // thrid was introduced in v13
2932 if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) return 1;
2933
2934 if (mailbox->i.minor_version < 15) return 0;
2935 // savedate was introduced in v15
2936 if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) return 1;
2937
2938 if (mailbox->i.minor_version < 16) return 0;
2939 // createdmodseq was introduced in v15
2940 if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) return 1;
2941
2942 return 0;
2943 }
2944
2945
crc_virtannot(struct mailbox * mailbox,const struct index_record * record)2946 static uint32_t crc_virtannot(struct mailbox *mailbox,
2947 const struct index_record *record)
2948 {
2949 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED)
2950 return 0;
2951
2952 uint32_t crc = 0;
2953 struct buf buf = BUF_INITIALIZER;
2954
2955 if (record->cid && mailbox->i.minor_version >= 13) {
2956 buf_printf(&buf, "%llx", record->cid);
2957 crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "thrid", "", &buf);
2958 buf_reset(&buf);
2959 }
2960
2961 if (record->savedate && mailbox->i.minor_version >= 15) {
2962 buf_printf(&buf, TIME_T_FMT, record->savedate);
2963 crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "savedate", "", &buf);
2964 buf_reset(&buf);
2965 }
2966
2967 if (record->createdmodseq && mailbox->i.minor_version >= 16) {
2968 buf_printf(&buf, "%llu", record->createdmodseq);
2969 crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf);
2970 buf_reset(&buf);
2971 }
2972
2973 buf_free(&buf);
2974 return crc;
2975 }
2976
mailbox_annot_changed(struct mailbox * mailbox,unsigned int uid,const char * entry,const char * userid,const struct buf * oldval,const struct buf * newval,int silent)2977 EXPORTED void mailbox_annot_changed(struct mailbox *mailbox,
2978 unsigned int uid,
2979 const char *entry,
2980 const char *userid,
2981 const struct buf *oldval,
2982 const struct buf *newval,
2983 int silent)
2984 {
2985 /* update sync_crc - NOTE, only per-message annotations count */
2986 if (uid) {
2987 /* check that the record isn't already expunged */
2988 struct index_record record;
2989 int r = mailbox_find_index_record(mailbox, uid, &record);
2990 if (r || record.internal_flags & FLAG_INTERNAL_EXPUNGED)
2991 return;
2992 if (!mailbox_is_virtannot(mailbox, entry)) {
2993 if (oldval->len)
2994 mailbox->i.synccrcs.annot ^= crc_annot(uid, entry, userid, oldval);
2995 if (newval->len)
2996 mailbox->i.synccrcs.annot ^= crc_annot(uid, entry, userid, newval);
2997 }
2998 }
2999
3000 if (!silent) {
3001 /* we are dirtying modseq for any annotation change */
3002 mailbox_modseq_dirty(mailbox);
3003 /* and we're dirtying foldermodseq if it's a mailbox level annotation */
3004 if (!uid) mboxlist_update_foldermodseq(mailbox->name, mailbox->i.highestmodseq);
3005 }
3006 /* we always dirty the quota */
3007 mailbox_quota_dirty(mailbox);
3008
3009 /* corruption prevention - check we don't go negative */
3010 if (mailbox->i.quota_annot_used > (quota_t)oldval->len)
3011 mailbox->i.quota_annot_used -= oldval->len;
3012 else
3013 mailbox->i.quota_annot_used = 0;
3014
3015 mailbox->i.quota_annot_used += newval->len;
3016 }
3017
calc_one_annot(const char * mboxname,uint32_t uid,const char * entry,const char * userid,const struct buf * value,const struct annotate_metadata * mdata,void * rock)3018 static int calc_one_annot(const char *mboxname __attribute__((unused)),
3019 uint32_t uid,
3020 const char *entry,
3021 const char *userid,
3022 const struct buf *value,
3023 const struct annotate_metadata *mdata __attribute__((unused)),
3024 void *rock)
3025 {
3026 struct annot_calc_rock *cr = (struct annot_calc_rock *)rock;
3027
3028 /* update sync_crc - NOTE, only per-message annotations count */
3029 if (uid && !mailbox_is_virtannot(cr->mailbox, entry))
3030 cr->annot ^= crc_annot(uid, entry, userid, value);
3031
3032 /* always count the size */
3033 cr->used += value->len;
3034
3035 return 0;
3036 }
3037
mailbox_annot_update_counts(struct mailbox * mailbox,const struct index_record * record,int is_add)3038 static void mailbox_annot_update_counts(struct mailbox *mailbox,
3039 const struct index_record *record,
3040 int is_add)
3041 {
3042 struct annot_calc_rock cr = { mailbox, 0, 0 };
3043
3044 /* expunged records don't count */
3045 if (record && record->internal_flags & FLAG_INTERNAL_EXPUNGED) return;
3046
3047 annotatemore_findall(mailbox->name, record ? record->uid : 0, /* all entries*/"*",
3048 /*modseq*/0, calc_one_annot, &cr, /*flags*/0);
3049
3050 if (record)
3051 mailbox->i.synccrcs.annot ^= cr.annot;
3052
3053 if (is_add)
3054 mailbox->i.quota_annot_used += cr.used;
3055 else {
3056 /* corruption prevention - check we don't go negative */
3057 if (mailbox->i.quota_annot_used > cr.used)
3058 mailbox->i.quota_annot_used -= cr.used;
3059 else
3060 mailbox->i.quota_annot_used = 0;
3061 }
3062 }
3063
3064 /*
3065 * Calculate a sync CRC for the entire @mailbox using CRC algorithm
3066 * version @vers, optionally forcing recalculation
3067 */
mailbox_synccrcs(struct mailbox * mailbox,int force)3068 EXPORTED struct synccrcs mailbox_synccrcs(struct mailbox *mailbox, int force)
3069 {
3070 annotate_state_t *astate = NULL;
3071 const message_t *msg;
3072 struct synccrcs crcs = { CRC_INIT_BASIC, CRC_INIT_ANNOT };
3073
3074 if (!force)
3075 return mailbox->i.synccrcs;
3076
3077 /* hold annotations DB open - failure to load is an error */
3078 if (mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, &astate))
3079 return crcs;
3080
3081 /* and make sure it stays locked for the whole process */
3082 annotate_state_begin(astate);
3083
3084 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED);
3085 while ((msg = mailbox_iter_step(iter))) {
3086 const struct index_record *record = msg_record(msg);
3087 crcs.basic ^= crc_basic(mailbox, record);
3088 crcs.annot ^= crc_virtannot(mailbox, record);
3089
3090 struct annot_calc_rock cr = { mailbox, 0, 0 };
3091 annotatemore_findall(mailbox->name, record->uid, /* all entries*/"*",
3092 /*modseq*/0, calc_one_annot, &cr, /*flags*/0);
3093
3094 crcs.annot ^= cr.annot;
3095 }
3096 mailbox_iter_done(&iter);
3097
3098 /* possibly upgrade the stored value */
3099 if (mailbox_index_islocked(mailbox, /*write*/1)) {
3100 mailbox->i.synccrcs = crcs;
3101 mailbox_index_dirty(mailbox);
3102 }
3103
3104 /* return the newly calculated CRC */
3105 return crcs;
3106 }
3107
mailbox_index_update_counts(struct mailbox * mailbox,const struct index_record * record,int is_add)3108 static void mailbox_index_update_counts(struct mailbox *mailbox,
3109 const struct index_record *record,
3110 int is_add)
3111 {
3112 mailbox_quota_dirty(mailbox);
3113 mailbox_index_dirty(mailbox);
3114 header_update_counts(&mailbox->i, record, is_add);
3115
3116 mailbox->i.synccrcs.basic ^= crc_basic(mailbox, record);
3117 mailbox->i.synccrcs.annot ^= crc_virtannot(mailbox, record);
3118 }
3119
mailbox_index_recalc(struct mailbox * mailbox)3120 EXPORTED int mailbox_index_recalc(struct mailbox *mailbox)
3121 {
3122 annotate_state_t *astate = NULL;
3123 const message_t *msg;
3124 int r = 0;
3125
3126 assert(mailbox_index_islocked(mailbox, 1));
3127
3128 /* cache the old used quota */
3129 mailbox_quota_dirty(mailbox);
3130 mailbox_index_dirty(mailbox);
3131
3132 mailbox->i.answered = 0;
3133 mailbox->i.flagged = 0;
3134 mailbox->i.deleted = 0;
3135 mailbox->i.unseen = 0;
3136 mailbox->i.exists = 0;
3137 mailbox->i.quota_mailbox_used = 0;
3138 mailbox->i.quota_annot_used = 0;
3139 mailbox->i.synccrcs.basic = CRC_INIT_BASIC;
3140 mailbox->i.synccrcs.annot = CRC_INIT_ANNOT;
3141
3142 /* mailbox level annotations */
3143 mailbox_annot_update_counts(mailbox, NULL, 1);
3144
3145 /* hold annotations DB open */
3146 r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, &astate);
3147 if (r) goto out;
3148
3149 /* and make sure it stays locked for the whole process */
3150 annotate_state_begin(astate);
3151
3152 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED);
3153 while ((msg = mailbox_iter_step(iter))) {
3154 const struct index_record *record = msg_record(msg);
3155 mailbox_index_update_counts(mailbox, record, 1);
3156 mailbox_annot_update_counts(mailbox, record, 1);
3157 }
3158 mailbox_iter_done(&iter);
3159
3160 out:
3161 return r;
3162 }
3163
3164 #ifdef WITH_DAV
mailbox_update_carddav(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3165 static int mailbox_update_carddav(struct mailbox *mailbox,
3166 const struct index_record *old,
3167 struct index_record *new)
3168 {
3169 struct carddav_db *carddavdb = NULL;
3170 struct param *param;
3171 struct body *body = NULL;
3172 struct carddav_data *cdata = NULL;
3173 const char *resource = NULL;
3174 int r = 0;
3175
3176 /* conditions in which there's nothing to do */
3177 if (!new) goto done;
3178
3179 /* phantom record - never really existed here */
3180 if (!old && (new->internal_flags & FLAG_INTERNAL_UNLINKED))
3181 goto done;
3182
3183 r = mailbox_cacherecord(mailbox, new);
3184 if (r) goto done;
3185
3186 /* Get resource URL from filename param in Content-Disposition header */
3187 message_read_bodystructure(new, &body);
3188 for (param = body->disposition_params; param; param = param->next) {
3189 if (!strcmp(param->attribute, "FILENAME")) {
3190 resource = param->value;
3191 }
3192 }
3193
3194 assert(resource);
3195
3196 carddavdb = mailbox_open_carddav(mailbox);
3197
3198 /* find existing record for this resource */
3199 carddav_lookup_resource(carddavdb, mailbox->name, resource, &cdata, /*tombstones*/1);
3200
3201 /* does it still come from this UID? */
3202 if (cdata->dav.imap_uid > new->uid) goto done;
3203
3204 if (new->internal_flags & FLAG_INTERNAL_UNLINKED) {
3205 /* is there an existing record? */
3206 if (!cdata->dav.imap_uid) goto done;
3207
3208 /* delete entry */
3209 r = carddav_delete(carddavdb, cdata->dav.rowid);
3210 }
3211 else if (cdata->dav.imap_uid == new->uid) {
3212 /* just a flag change on an existing record */
3213 struct vparse_card *vcard = record_to_vcard(mailbox, new);
3214 int ispinned = (new->system_flags & FLAG_FLAGGED) ? 1 : 0;
3215
3216 if (!vcard || !vcard->objects) {
3217 syslog(LOG_ERR, "record_to_vcard failed for record %u:%s",
3218 cdata->dav.imap_uid, mailbox->name);
3219 r = IMAP_MAILBOX_BADFORMAT; // XXX better error?
3220 vparse_free_card(vcard);
3221 goto done;
3222 }
3223
3224 cdata->dav.modseq = new->modseq;
3225 cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3226 r = carddav_writecard(carddavdb, cdata, vcard->objects, ispinned);
3227 vparse_free_card(vcard);
3228 }
3229 else {
3230 /* Load message containing the resource and parse vcard data */
3231 struct vparse_card *vcard = record_to_vcard(mailbox, new);
3232 int ispinned = (new->system_flags & FLAG_FLAGGED) ? 1 : 0;
3233
3234 if (!vcard || !vcard->objects) {
3235 syslog(LOG_ERR, "record_to_vcard failed for record %u:%s",
3236 cdata->dav.imap_uid, mailbox->name);
3237 r = IMAP_MAILBOX_BADFORMAT; // XXX better error?
3238 vparse_free_card(vcard);
3239 goto done;
3240 }
3241
3242 /* Create mapping entry from resource name to UID */
3243 cdata->dav.mailbox = mailbox->name;
3244 cdata->dav.resource = resource;
3245 cdata->dav.imap_uid = new->uid;
3246 cdata->dav.modseq = new->modseq;
3247 cdata->dav.createdmodseq = new->createdmodseq;
3248 cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3249
3250 if (!cdata->dav.creationdate)
3251 cdata->dav.creationdate = new->internaldate;
3252
3253 r = carddav_writecard(carddavdb, cdata, vcard->objects, ispinned);
3254
3255 vparse_free_card(vcard);
3256 }
3257
3258 done:
3259 message_free_body(body);
3260 free(body);
3261
3262 return r;
3263 }
3264
mailbox_update_caldav(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3265 static int mailbox_update_caldav(struct mailbox *mailbox,
3266 const struct index_record *old,
3267 struct index_record *new)
3268 {
3269 struct caldav_db *caldavdb = NULL;
3270 struct param *param;
3271 struct body *body = NULL;
3272 struct caldav_data *cdata = NULL;
3273 const char *resource = NULL;
3274 const char *sched_tag = NULL;
3275 unsigned tzbyref = 0, shared = 0;
3276 int r = 0;
3277
3278 /* conditions in which there's nothing to do */
3279 if (!new) goto done;
3280
3281 /* phantom record - never really existed at all */
3282 if (!old && (new->internal_flags & FLAG_INTERNAL_UNLINKED))
3283 goto done;
3284
3285 r = mailbox_cacherecord(mailbox, new);
3286 if (r) goto done;
3287
3288 /* get resource URL from filename param in Content-Disposition header */
3289 message_read_bodystructure(new, &body);
3290 for (param = body->disposition_params; param; param = param->next) {
3291 if (!strcmp(param->attribute, "FILENAME")) {
3292 resource = param->value;
3293 }
3294 else if (!strcmp(param->attribute, "SCHEDULE-TAG")) {
3295 sched_tag = param->value;
3296 }
3297 else if (!strcmp(param->attribute, "TZ-BY-REF")) {
3298 tzbyref = !strcasecmp(param->value, "TRUE");
3299 }
3300 else if (!strcmp(param->attribute, "PER-USER-DATA")) {
3301 shared = !strcasecmp(param->value, "TRUE");
3302 }
3303 }
3304
3305 caldavdb = mailbox_open_caldav(mailbox);
3306
3307 /* Find existing record for this resource */
3308 caldav_lookup_resource(caldavdb, mailbox->name, resource, &cdata, /*tombstones*/1);
3309
3310 /* has this record already been replaced? Don't write anything */
3311 if (cdata->dav.imap_uid > new->uid) goto done;
3312
3313 if (new->internal_flags & FLAG_INTERNAL_UNLINKED) {
3314 /* is there an existing record? */
3315 if (!cdata->dav.imap_uid) goto done;
3316
3317 /* remove associated alarms */
3318 caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid);
3319
3320 /* delete entry */
3321 r = caldav_delete(caldavdb, cdata->dav.rowid);
3322 }
3323 else if (cdata->dav.imap_uid == new->uid) {
3324 if (new->internal_flags & FLAG_INTERNAL_EXPUNGED) {
3325 /* remove associated alarms */
3326 caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid);
3327 }
3328 else if (!new->silent) {
3329 /* make sure record is up to date - see add below for description of
3330 * why we don't touch silent records */
3331 caldav_alarm_touch_record(mailbox, new);
3332 }
3333
3334 /* just a flags update to an existing record */
3335 cdata->dav.modseq = new->modseq;
3336 cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3337 r = caldav_write(caldavdb, cdata);
3338 }
3339 else {
3340 /* Load message containing the resource and parse ical data */
3341 icalcomponent *ical = record_to_ical(mailbox, new, NULL);
3342
3343 if (!ical) {
3344 syslog(LOG_ERR, "record_to_ical failed for record %u:%s",
3345 cdata->dav.imap_uid, mailbox->name);
3346 r = IMAP_MAILBOX_BADFORMAT; // XXX better error?
3347 goto done;
3348 }
3349
3350 /* remove old ones */
3351 if (cdata->dav.imap_uid) {
3352 r = caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid);
3353 if (r) goto alarmdone;
3354 }
3355
3356 cdata->dav.creationdate = new->internaldate;
3357 cdata->dav.mailbox = mailbox->name;
3358 cdata->dav.imap_uid = new->uid;
3359 cdata->dav.modseq = new->modseq;
3360 cdata->dav.createdmodseq = new->createdmodseq;
3361 cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3362 cdata->dav.resource = resource;
3363 cdata->sched_tag = sched_tag;
3364 cdata->comp_flags.tzbyref = tzbyref;
3365 cdata->comp_flags.shared = shared;
3366
3367 /* add new ones unless this record is expunged */
3368 /* we need to skip silent records (replication)
3369 * because the lastalarm annotation won't be set yet -
3370 * instead, we have an explicit sync from the annotation
3371 * which is done after the annotations are written in sync_support.c */
3372 if (cdata->dav.alive && !new->silent) {
3373 r = caldav_alarm_add_record(mailbox, new, ical);
3374 if (r) goto alarmdone;
3375 }
3376
3377 r = caldav_writeentry(caldavdb, cdata, ical);
3378
3379 alarmdone:
3380 icalcomponent_free(ical);
3381 }
3382
3383 done:
3384 message_free_body(body);
3385 free(body);
3386
3387 return r;
3388 }
3389
mailbox_update_webdav(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3390 static int mailbox_update_webdav(struct mailbox *mailbox,
3391 const struct index_record *old,
3392 struct index_record *new)
3393 {
3394 struct webdav_db *webdavdb = NULL;
3395 struct param *param;
3396 struct body *body = NULL;
3397 struct webdav_data *wdata = NULL;
3398 const char *resource = NULL;
3399 int r = 0;
3400
3401 /* conditions in which there's nothing to do */
3402 if (!new) goto done;
3403
3404 /* phantom record - never really existed here */
3405 if (!old && new->internal_flags & FLAG_INTERNAL_EXPUNGED)
3406 goto done;
3407
3408 r = mailbox_cacherecord(mailbox, new);
3409 if (r) goto done;
3410
3411 /* Get resource URL from filename param in Content-Disposition header */
3412 message_read_bodystructure(new, &body);
3413 for (param = body->disposition_params; param; param = param->next) {
3414 if (!strcmp(param->attribute, "FILENAME")) {
3415 resource = param->value;
3416 }
3417 }
3418
3419 webdavdb = mailbox_open_webdav(mailbox);
3420
3421 /* Find existing record for this resource */
3422 webdav_lookup_resource(webdavdb, mailbox->name, resource, &wdata, /*tombstones*/1);
3423
3424 /* if updated by a newer UID, skip - this record doesn't refer to the current item */
3425 if (wdata->dav.imap_uid > new->uid) goto done;
3426
3427 if (new->internal_flags & FLAG_INTERNAL_UNLINKED) {
3428 /* is there an existing record? */
3429 if (!wdata->dav.imap_uid) goto done;
3430
3431 /* delete entry */
3432 r = webdav_delete(webdavdb, wdata->dav.rowid);
3433 }
3434 else if (wdata->dav.imap_uid == new->uid) {
3435 /* just a flags update to an existing record */
3436 wdata->dav.modseq = new->modseq;
3437 wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3438 wdata->ref_count *= wdata->dav.alive;
3439 r = webdav_write(webdavdb, wdata);
3440 }
3441 else {
3442 struct buf msg_buf = BUF_INITIALIZER;
3443 struct message_guid guid;
3444
3445 /* Load message containing the resource */
3446 r = mailbox_map_record(mailbox, new, &msg_buf);
3447 if (r) goto done;
3448
3449 /* Calculate GUID for body content only */
3450 message_guid_generate(&guid, buf_base(&msg_buf) + new->header_size,
3451 buf_len(&msg_buf) - new->header_size);
3452 buf_free(&msg_buf);
3453
3454 wdata->dav.creationdate = new->internaldate;
3455 wdata->dav.mailbox = mailbox->name;
3456 wdata->dav.imap_uid = new->uid;
3457 wdata->dav.modseq = new->modseq;
3458 wdata->dav.createdmodseq = new->createdmodseq;
3459 wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
3460 wdata->ref_count *= wdata->dav.alive;
3461 wdata->dav.resource = resource;
3462 wdata->filename = body->description;
3463 wdata->type = lcase(body->type);
3464 wdata->subtype = lcase(body->subtype);
3465 wdata->res_uid = message_guid_encode(&guid);
3466
3467 r = webdav_write(webdavdb, wdata);
3468 }
3469
3470 done:
3471 if (body) {
3472 message_free_body(body);
3473 free(body);
3474 }
3475
3476 return r;
3477 }
3478
mailbox_update_dav(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3479 static int mailbox_update_dav(struct mailbox *mailbox,
3480 const struct index_record *old,
3481 struct index_record *new)
3482 {
3483 /* never have DAV on deleted mailboxes */
3484 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
3485 return 0;
3486
3487 if (mailbox->mbtype & MBTYPE_ADDRESSBOOK)
3488 return mailbox_update_carddav(mailbox, old, new);
3489 if (mailbox->mbtype & MBTYPE_CALENDAR)
3490 return mailbox_update_caldav(mailbox, old, new);
3491 if (mailbox->mbtype & MBTYPE_COLLECTION)
3492 return mailbox_update_webdav(mailbox, old, new);
3493
3494 return 0;
3495 }
3496
mailbox_commit_dav(struct mailbox * mailbox)3497 static int mailbox_commit_dav(struct mailbox *mailbox)
3498 {
3499 int r;
3500
3501 if (mailbox->local_caldav) {
3502 r = caldav_commit(mailbox->local_caldav);
3503 caldav_close(mailbox->local_caldav);
3504 mailbox->local_caldav = NULL;
3505 if (r) return r;
3506 }
3507
3508 if (mailbox->local_carddav) {
3509 r = carddav_commit(mailbox->local_carddav);
3510 carddav_close(mailbox->local_carddav);
3511 mailbox->local_carddav = NULL;
3512 if (r) return r;
3513 }
3514
3515 if (mailbox->local_webdav) {
3516 r = webdav_commit(mailbox->local_webdav);
3517 webdav_close(mailbox->local_webdav);
3518 mailbox->local_webdav = NULL;
3519 if (r) return r;
3520 }
3521
3522 return 0;
3523 }
3524
mailbox_abort_dav(struct mailbox * mailbox)3525 static int mailbox_abort_dav(struct mailbox *mailbox)
3526 {
3527 int r;
3528
3529 if (mailbox->local_caldav) {
3530 r = caldav_abort(mailbox->local_caldav);
3531 caldav_close(mailbox->local_caldav);
3532 mailbox->local_caldav = NULL;
3533 if (r) return r;
3534 }
3535
3536 if (mailbox->local_carddav) {
3537 r = carddav_abort(mailbox->local_carddav);
3538 carddav_close(mailbox->local_carddav);
3539 mailbox->local_carddav = NULL;
3540 if (r) return r;
3541 }
3542
3543 if (mailbox->local_webdav) {
3544 r = webdav_abort(mailbox->local_webdav);
3545 webdav_close(mailbox->local_webdav);
3546 mailbox->local_webdav = NULL;
3547 if (r) return r;
3548 }
3549
3550 return 0;
3551 }
3552
3553 #endif // WITH_DAV
3554
3555 #ifdef WITH_JMAP
mailbox_update_email_alarms(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3556 static int mailbox_update_email_alarms(struct mailbox *mailbox,
3557 const struct index_record *old,
3558 struct index_record *new)
3559 {
3560 int r = 0;
3561
3562 if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS))
3563 return 0;
3564
3565 /* never have alarms on deleted mailboxes */
3566 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
3567 return 0;
3568
3569 /* remove associated alarms if deleted */
3570 if (!new || new->internal_flags & FLAG_INTERNAL_EXPUNGED) {
3571 r = caldav_alarm_delete_record(mailbox->name, new->uid);
3572 }
3573
3574 /* remove associated alarms if canceled or final */
3575 else if ((mailbox->mbtype & MBTYPE_SUBMISSION) &&
3576 (new->system_flags & (FLAG_FLAGGED | FLAG_ANSWERED))) {
3577 r = caldav_alarm_delete_record(mailbox->name, new->uid);
3578 }
3579
3580 /* touch or create otherwise */
3581 else if (old && (old->uid == new->uid)) {
3582 r = caldav_alarm_touch_record(mailbox, new);
3583 }
3584 else {
3585 r = caldav_alarm_add_record(mailbox, new, NULL);
3586 }
3587
3588 return r;
3589 }
3590
mailbox_add_email_alarms(struct mailbox * mailbox)3591 EXPORTED int mailbox_add_email_alarms(struct mailbox *mailbox)
3592 {
3593 const message_t *msg;
3594 int r = 0;
3595
3596 if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS))
3597 return 0;
3598
3599 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
3600 return 0;
3601
3602 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
3603 while ((msg = mailbox_iter_step(iter))) {
3604 const struct index_record *record = msg_record(msg);
3605 struct index_record copyrecord = *record;
3606 r = mailbox_update_email_alarms(mailbox, NULL, ©record);
3607 if (r) break;
3608 /* in THEORY there maybe changes here that we should be saving... */
3609 }
3610 mailbox_iter_done(&iter);
3611
3612 return r;
3613 }
3614 #endif // WITH_JMAP
3615
mailbox_get_cstate(struct mailbox * mailbox)3616 EXPORTED struct conversations_state *mailbox_get_cstate(struct mailbox *mailbox)
3617 {
3618 if (!mailbox_has_conversations(mailbox))
3619 return NULL;
3620
3621 mailbox_lock_conversations(mailbox, mailbox->is_readonly ? LOCK_SHARED : LOCK_EXCLUSIVE);
3622
3623 return conversations_get_mbox(mailbox->name);
3624 }
3625
mailbox_update_conversations(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3626 static int mailbox_update_conversations(struct mailbox *mailbox,
3627 const struct index_record *old,
3628 struct index_record *new)
3629 {
3630 struct conversations_state *cstate = mailbox_get_cstate(mailbox);
3631
3632 if (!cstate)
3633 return 0;
3634
3635 /* handle unlinked items as if they didn't exist */
3636 if (old && (old->internal_flags & FLAG_INTERNAL_UNLINKED)) old = NULL;
3637 if (new && (new->internal_flags & FLAG_INTERNAL_UNLINKED)) new = NULL;
3638
3639 if (!old && !new)
3640 return 0;
3641
3642 return conversations_update_record(cstate, mailbox, old, new, /*allowrenumber*/1);
3643 }
3644
3645
mailbox_get_xconvmodseq(struct mailbox * mailbox,modseq_t * modseqp)3646 EXPORTED int mailbox_get_xconvmodseq(struct mailbox *mailbox, modseq_t *modseqp)
3647 {
3648 conv_status_t status = CONV_STATUS_INIT;
3649 int r;
3650
3651 if (modseqp)
3652 *modseqp = 0;
3653
3654 struct conversations_state *cstate = mailbox_get_cstate(mailbox);
3655 if (!cstate) return 0;
3656
3657 r = conversation_getstatus(cstate, mailbox->name, &status);
3658 if (r) return r;
3659
3660 *modseqp = status.threadmodseq;
3661
3662 return 0;
3663 }
3664
3665 /* Used in replication */
mailbox_update_xconvmodseq(struct mailbox * mailbox,modseq_t newmodseq,int force)3666 EXPORTED int mailbox_update_xconvmodseq(struct mailbox *mailbox, modseq_t newmodseq, int force)
3667 {
3668 conv_status_t status = CONV_STATUS_INIT;
3669 int r;
3670
3671 struct conversations_state *cstate = mailbox_get_cstate(mailbox);
3672 if (!cstate) return 0;
3673
3674 r = conversation_getstatus(cstate, mailbox->name, &status);
3675 if (r) return r;
3676
3677 if (newmodseq > status.threadmodseq || (force && newmodseq < status.threadmodseq)) {
3678 status.threadmodseq = newmodseq;
3679 r = conversation_setstatus(cstate, mailbox->name, &status);
3680 }
3681
3682 return r;
3683 }
3684
3685 /* NOTE: maybe make this able to return error codes if we have
3686 * support for transactional mailbox updates later. For now,
3687 * we expect callers to have already done all sanity checking */
mailbox_update_indexes(struct mailbox * mailbox,const struct index_record * old,struct index_record * new)3688 static int mailbox_update_indexes(struct mailbox *mailbox,
3689 const struct index_record *old,
3690 struct index_record *new)
3691 {
3692 int r = 0;
3693 #ifdef WITH_DAV
3694 r = mailbox_update_dav(mailbox, old, new);
3695 if (r) return r;
3696 #endif
3697
3698 #ifdef WITH_JMAP
3699 r = mailbox_update_email_alarms(mailbox, old, new);
3700 if (r) return r;
3701 #endif
3702
3703 r = mailbox_update_conversations(mailbox, old, new);
3704 if (r) return r;
3705
3706 /* NOTE - we do these last, once the counts are updated */
3707
3708 if (old)
3709 mailbox_index_update_counts(mailbox, old, 0);
3710 if (new)
3711 mailbox_index_update_counts(mailbox, new, 1);
3712
3713 return 0;
3714 }
3715
mailbox_reload_index_record(struct mailbox * mailbox,struct index_record * record)3716 EXPORTED int mailbox_reload_index_record(struct mailbox *mailbox,
3717 struct index_record *record)
3718 {
3719 if (record->recno)
3720 return mailbox_read_index_record(mailbox, record->recno, record);
3721 else
3722 return mailbox_find_index_record(mailbox, record->uid, record);
3723 }
3724
3725 /*
3726 * Rewrite an index record in a mailbox - updates all
3727 * necessary tracking fields automatically.
3728 */
mailbox_rewrite_index_record(struct mailbox * mailbox,struct index_record * record)3729 EXPORTED int mailbox_rewrite_index_record(struct mailbox *mailbox,
3730 struct index_record *record)
3731 {
3732 int r;
3733 struct index_record oldrecord;
3734 int expunge_mode = config_getenum(IMAPOPT_EXPUNGE_MODE);
3735 int immediate = (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE ||
3736 expunge_mode == IMAP_ENUM_EXPUNGE_MODE_SEMIDELAYED ||
3737 mailbox->i.minor_version < 12);
3738 int changeflags = 0;
3739
3740 assert(mailbox_index_islocked(mailbox, 1));
3741 assert(record->recno > 0 &&
3742 record->recno <= mailbox->i.num_records);
3743
3744 r = mailbox_read_index_record(mailbox, record->recno, &oldrecord);
3745 if (r) {
3746 syslog(LOG_ERR, "IOERROR: re-reading: %s %u",
3747 mailbox->name, record->uid);
3748 return r;
3749 }
3750 mailbox_read_basecid(mailbox, &oldrecord);
3751
3752 /* OK, we need to decide how to handle basecid. Three cases here:
3753 * 1) basecid is already set - keep it
3754 * 2) basecid is zero, but the system flag says it should be readable, set it from oldrecord
3755 * 3) basecid is zero, system flag is zero - keep zero
3756 * we only need code for case #2
3757 */
3758 if (!record->basecid && (record->internal_flags & FLAG_INTERNAL_SPLITCONVERSATION))
3759 record->basecid = oldrecord.basecid;
3760
3761 if (oldrecord.internal_flags & FLAG_INTERNAL_EXPUNGED)
3762 changeflags |= CHANGE_WASEXPUNGED;
3763 if (oldrecord.internal_flags & FLAG_INTERNAL_UNLINKED)
3764 changeflags |= CHANGE_WASUNLINKED;
3765
3766 /* the UID has to match, of course, for it to be the same
3767 * record. XXX - test fields like "internaldate", etc here
3768 * too? Maybe replication should be more strict about it */
3769 assert(record->uid == oldrecord.uid);
3770 assert(message_guid_equal(&oldrecord.guid, &record->guid));
3771
3772 if (oldrecord.internal_flags & FLAG_INTERNAL_EXPUNGED) {
3773 /* it is a sin to unexpunge a message. unexpunge.c copies
3774 * the data from the old record and appends it with a new
3775 * UID, which is righteous in the eyes of the IMAP client */
3776 assert(record->internal_flags & FLAG_INTERNAL_EXPUNGED);
3777 }
3778
3779 if (oldrecord.internal_flags & FLAG_INTERNAL_ARCHIVED) {
3780 /* it is also a sin to unarchive a message, except in the
3781 * the very odd case of a reconstruct. So let's see about
3782 * that */
3783 if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED))
3784 syslog(LOG_ERR, "IOERROR: bogus removal of archived flag for %s %u",
3785 mailbox->name, record->uid);
3786 }
3787
3788 /* handle immediate expunges here... */
3789 if (immediate && (record->internal_flags & FLAG_INTERNAL_EXPUNGED)) {
3790 record->internal_flags |= FLAG_INTERNAL_UNLINKED | FLAG_INTERNAL_NEEDS_CLEANUP;
3791 }
3792
3793 /* make sure highestmodseq gets updated unless we're
3794 * being silent about it (i.e. marking an already EXPUNGED
3795 * message as UNLINKED, or just updating the content_lines
3796 * field or cache_offset) */
3797 if (record->silent) {
3798 mailbox_index_dirty(mailbox);
3799 }
3800 else {
3801 mailbox_modseq_dirty(mailbox);
3802 record->modseq = mailbox->i.highestmodseq;
3803 record->last_updated = mailbox->last_updated;
3804 }
3805 assert(record->modseq >= oldrecord.modseq);
3806
3807 if (record->internal_flags & FLAG_INTERNAL_UNLINKED) {
3808 /* mark required actions */
3809 if (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE
3810 || mailbox->i.minor_version < 12)
3811 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
3812 mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK;
3813 }
3814 else {
3815 /* rewrite the cache record if required anyway */
3816 r = mailbox_append_cache(mailbox, record);
3817 if (r) return r;
3818 }
3819
3820 r = mailbox_update_indexes(mailbox, &oldrecord, record);
3821 if (r) return r;
3822
3823 if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(changeflags & CHANGE_WASEXPUNGED)) {
3824 if (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated)
3825 mailbox->i.first_expunged = record->last_updated;
3826 mailbox_annot_update_counts(mailbox, &oldrecord, 0);
3827 }
3828
3829 r = _store_change(mailbox, record, changeflags);
3830 if (r) return r;
3831
3832 if (config_auditlog) {
3833 char oldflags[FLAGMAPSTR_MAXLEN], sysflags[FLAGMAPSTR_MAXLEN];
3834 flags_to_str(&oldrecord, oldflags);
3835 flags_to_str(record, sysflags);
3836 syslog(LOG_NOTICE, "auditlog: touched sessionid=<%s> "
3837 "mailbox=<%s> uniqueid=<%s> uid=<%u> guid=<%s> cid=<%s> "
3838 "modseq=<" MODSEQ_FMT "> oldflags=<%s> sysflags=<%s>",
3839 session_id(), mailbox->name, mailbox->uniqueid,
3840 record->uid, message_guid_encode(&record->guid),
3841 conversation_id_encode(record->cid), record->modseq,
3842 oldflags, sysflags);
3843 }
3844
3845 /* expunged tracking */
3846 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED && (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated))
3847 mailbox->i.first_expunged = record->last_updated;
3848
3849 return 0;
3850 }
3851
3852 /* append a single message to a mailbox - also updates everything
3853 * automatically. These two functions are the ONLY way to modify
3854 * the contents or tracking fields of a message */
mailbox_append_index_record(struct mailbox * mailbox,struct index_record * record)3855 EXPORTED int mailbox_append_index_record(struct mailbox *mailbox,
3856 struct index_record *record)
3857 {
3858 int r;
3859 struct utimbuf settime;
3860 uint32_t changeflags = CHANGE_ISAPPEND;
3861
3862 assert(mailbox_index_islocked(mailbox, 1));
3863
3864 /* Append MUST be a higher UID than any we've yet seen */
3865 assert(record->uid > mailbox->i.last_uid)
3866
3867 /* Append MUST have a message with data */
3868 assert(record->size);
3869
3870 /* GUID must not be null */
3871 assert(!message_guid_isnull(&record->guid));
3872
3873 /* belt AND suspenders - check the previous record too */
3874 if (mailbox->i.num_records) {
3875 struct index_record prev;
3876 r = mailbox_read_index_record(mailbox, mailbox->i.num_records, &prev);
3877 if (r) return r;
3878 assert(prev.uid <= mailbox->i.last_uid);
3879 if (message_guid_equal(&prev.guid, &record->guid)) {
3880 syslog(LOG_INFO, "%s: same message appears twice %u %u",
3881 mailbox->name, prev.uid, record->uid);
3882 /* but it's OK, we won't reject it */
3883 }
3884 }
3885
3886 if (!record->internaldate)
3887 record->internaldate = time(NULL);
3888 if (!record->gmtime)
3889 record->gmtime = record->internaldate;
3890 if (!record->sentdate) {
3891 struct tm *tm = localtime(&record->internaldate);
3892 /* truncate to the day */
3893 tm->tm_sec = 0;
3894 tm->tm_min = 0;
3895 tm->tm_hour = 0;
3896 record->sentdate = mktime(tm);
3897 }
3898
3899 /* update the highestmodseq if needed */
3900 if (record->silent) {
3901 mailbox_index_dirty(mailbox);
3902 }
3903 else {
3904 mailbox_modseq_dirty(mailbox);
3905 record->modseq = mailbox->i.highestmodseq;
3906 if (!record->createdmodseq || record->createdmodseq > record->modseq)
3907 record->createdmodseq = record->modseq;
3908 record->last_updated = mailbox->last_updated;
3909 if (!record->savedate) {
3910 // store the time of actual append if requested
3911 record->savedate = mailbox->last_updated;
3912 }
3913 }
3914
3915 int object_storage_enabled = 0 ;
3916 #if defined ENABLE_OBJECTSTORE
3917 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
3918 #endif
3919
3920 if (!(record->internal_flags & FLAG_INTERNAL_UNLINKED)) {
3921 /* make the file timestamp correct */
3922 settime.actime = settime.modtime = record->internaldate;
3923 if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) // mabe there is no file in directory.
3924 if (utime(mailbox_record_fname(mailbox, record), &settime) == -1)
3925 return IMAP_IOERROR;
3926
3927 /* write the cache record before buffering the message, it
3928 * will set the cache_offset field. */
3929 r = mailbox_append_cache(mailbox, record);
3930 if (r) return r;
3931 }
3932
3933 r = mailbox_update_indexes(mailbox, NULL, record);
3934 if (r) return r;
3935
3936 record->recno = mailbox->i.num_records + 1;
3937
3938 mailbox->i.last_uid = record->uid;
3939 mailbox->i.num_records = record->recno;
3940
3941 r = _store_change(mailbox, record, changeflags);
3942 if (r) return r;
3943
3944 /* expunged tracking */
3945 if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated))
3946 mailbox->i.first_expunged = record->last_updated;
3947
3948 return 0;
3949 }
3950
mailbox_cleanup_uid(struct mailbox * mailbox,uint32_t uid,const char * flagstr)3951 EXPORTED void mailbox_cleanup_uid(struct mailbox *mailbox, uint32_t uid, const char *flagstr)
3952 {
3953 const char *spoolfname = mailbox_spool_fname(mailbox, uid);
3954 const char *archivefname = mailbox_archive_fname(mailbox, uid);
3955
3956 if (unlink(spoolfname) == 0) {
3957 if (config_auditlog) {
3958 syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> "
3959 "mailbox=<%s> uniqueid=<%s> uid=<%u> sysflags=<%s>",
3960 session_id(), mailbox->name, mailbox->uniqueid,
3961 uid, flagstr);
3962 }
3963 }
3964
3965 if (strcmp(spoolfname, archivefname)) {
3966 if (unlink(archivefname) == 0) {
3967 if (config_auditlog) {
3968 syslog(LOG_NOTICE, "auditlog: unlinkarchive sessionid=<%s> "
3969 "mailbox=<%s> uniqueid=<%s> uid=<%u> sysflags=<%s>",
3970 session_id(), mailbox->name, mailbox->uniqueid,
3971 uid, flagstr);
3972 }
3973 }
3974 }
3975 }
3976
mailbox_record_cleanup(struct mailbox * mailbox,struct index_record * record)3977 static void mailbox_record_cleanup(struct mailbox *mailbox,
3978 struct index_record *record)
3979 {
3980 #if defined ENABLE_OBJECTSTORE
3981 if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) {
3982 /* we always remove the spool file here, because we've archived it */
3983 if (record->system_flags & FLAG_INTERNAL_ARCHIVED)
3984 unlink(spoolfname);
3985
3986 /* if the record is also deleted, we remove the objectstore copy */
3987 if (record->system_flags & FLAG_INTERNAL_UNLINKED)
3988 objectstore_delete(mailbox, record);
3989
3990 return;
3991 }
3992 #endif
3993
3994 if (record->internal_flags & FLAG_INTERNAL_UNLINKED) {
3995 char flagstr[FLAGMAPSTR_MAXLEN];
3996 flags_to_str(record, flagstr);
3997
3998 /* remove both files */
3999 mailbox_cleanup_uid(mailbox, record->uid, flagstr);
4000
4001 int r = mailbox_get_annotate_state(mailbox, record->uid, NULL);
4002 if (r) {
4003 syslog(LOG_ERR, "IOERROR: failed to open annotations %s %u: %s",
4004 mailbox->name, record->uid, error_message(r));
4005 }
4006
4007 r = annotate_msg_cleanup(mailbox, record->uid);
4008 if (r) {
4009 syslog(LOG_ERR, "IOERROR: failed to cleanup annotations %s %u: %s",
4010 mailbox->name, record->uid, error_message(r));
4011 }
4012
4013 return;
4014 }
4015
4016 /* file is still alive - check if there's anything to clean up */
4017
4018 const char *spoolfname = mailbox_spool_fname(mailbox, record->uid);
4019 const char *archivefname = mailbox_archive_fname(mailbox, record->uid);
4020
4021 /* don't cleanup if it's the same file! */
4022 if (strcmp(spoolfname, archivefname)) {
4023 if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) {
4024 /* XXX - stat to make sure the other file exists first? - we mostly
4025 * trust that we didn't do stupid things everywhere else, so maybe not */
4026 unlink(spoolfname);
4027 }
4028
4029 else {
4030 unlink(archivefname);
4031 }
4032 }
4033 }
4034
4035 /* need a mailbox exclusive lock, we're removing files */
mailbox_index_unlink(struct mailbox * mailbox)4036 static int mailbox_index_unlink(struct mailbox *mailbox)
4037 {
4038 syslog(LOG_INFO, "Unlinking files in mailbox %s", mailbox->name);
4039
4040 /* NOTE: this gets called for two different cases:
4041 * 1) file is actually ready for unlinking (immediate expunge or
4042 * cyr_expire).
4043 * 2) file has been archived/unarchived, and the other one needs
4044 * to be removed.
4045 */
4046 const message_t *msg;
4047 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, 0);
4048 while ((msg = mailbox_iter_step(iter))) {
4049 const struct index_record *record = msg_record(msg);
4050 /* still gotta check for FLAG_INTERNAL_UNLINKED, because it may have been
4051 * created by old code. Woot */
4052 if ((record->internal_flags & FLAG_INTERNAL_NEEDS_CLEANUP) ||
4053 record->internal_flags & FLAG_INTERNAL_UNLINKED) {
4054 struct index_record copyrecord = *record;
4055 mailbox_record_cleanup(mailbox, ©record);
4056 copyrecord.internal_flags &= ~FLAG_INTERNAL_NEEDS_CLEANUP;
4057 copyrecord.silent = 1;
4058 /* XXX - error handling */
4059 mailbox_rewrite_index_record(mailbox, ©record);
4060 }
4061 }
4062 mailbox_iter_done(&iter);
4063
4064 /* need to clear the flag, even if nothing needed unlinking! */
4065 mailbox_index_dirty(mailbox);
4066 mailbox->i.options &= ~OPT_MAILBOX_NEEDS_UNLINK;
4067 mailbox_commit(mailbox);
4068
4069 return 0;
4070 }
4071
4072
mailbox_repack_setup(struct mailbox * mailbox,int version,struct mailbox_repack ** repackptr)4073 static int mailbox_repack_setup(struct mailbox *mailbox, int version,
4074 struct mailbox_repack **repackptr)
4075 {
4076 struct mailbox_repack *repack = xzmalloc(sizeof(struct mailbox_repack));
4077 const char *fname;
4078 indexbuffer_t ibuf;
4079 unsigned char *buf = ibuf.buf;
4080 int n;
4081
4082 /* if we're changing version at all, recalculate counts up-front */
4083 if (version != mailbox->i.minor_version) {
4084 /* NOTE: this maps in annot_state in mailbox, which will get copied
4085 * into newmailbox below, which means all the annotate stuff later
4086 * is going to be good! The counters will be updated in the original
4087 * mailbox index, but that's OK because we recalc everything at the
4088 * end */
4089 int r = mailbox_index_recalc(mailbox);
4090 if (r) goto fail;
4091 }
4092
4093 /* init */
4094 repack->mailbox = mailbox;
4095 repack->crcs = mailbox->i.synccrcs;
4096 repack->newmailbox = *mailbox; // struct copy
4097 repack->newmailbox.index_fd = -1;
4098
4099 /* new files */
4100 fname = mailbox_meta_newfname(mailbox, META_INDEX);
4101 repack->newmailbox.index_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
4102 if (repack->newmailbox.index_fd == -1) {
4103 syslog(LOG_ERR, "IOERROR: failed to create %s: %m", fname);
4104 goto fail;
4105 }
4106
4107 /* update the generation number */
4108 repack->newmailbox.i.generation_no++;
4109
4110 /* track the version number */
4111 repack->newmailbox.i.minor_version = version;
4112 switch (version) {
4113 case 6:
4114 repack->newmailbox.i.start_offset = 76;
4115 repack->newmailbox.i.record_size = 60;
4116 break;
4117 case 7:
4118 repack->newmailbox.i.start_offset = 76;
4119 repack->newmailbox.i.record_size = 72;
4120 break;
4121 case 8:
4122 repack->newmailbox.i.start_offset = 92;
4123 repack->newmailbox.i.record_size = 80;
4124 break;
4125 case 9:
4126 repack->newmailbox.i.start_offset = 96;
4127 repack->newmailbox.i.record_size = 80;
4128 break;
4129 case 10:
4130 repack->newmailbox.i.start_offset = 96;
4131 repack->newmailbox.i.record_size = 88;
4132 break;
4133 /* 11 was FastMail internal */
4134 case 12:
4135 repack->newmailbox.i.start_offset = 128;
4136 repack->newmailbox.i.record_size = 96;
4137 break;
4138 case 13:
4139 repack->newmailbox.i.start_offset = 128;
4140 repack->newmailbox.i.record_size = 104;
4141 break;
4142 case 14:
4143 /* version 15 just repurposed a field */
4144 case 15:
4145 repack->newmailbox.i.start_offset = 160;
4146 repack->newmailbox.i.record_size = 104;
4147 break;
4148 case 16:
4149 repack->newmailbox.i.start_offset = 160;
4150 repack->newmailbox.i.record_size = 112;
4151 break;
4152 default:
4153 fatal("index version not supported", EX_SOFTWARE);
4154 }
4155
4156 /* upgrades or downgrades across version 12 boundary? Sort out seen state */
4157 if (version >= 12 && mailbox->i.minor_version < 12) {
4158 /* we need to read the current seen state for the owner */
4159 struct seendata sd = SEENDATA_INITIALIZER;
4160 int r = IMAP_MAILBOX_NONEXISTENT;
4161 if (mailbox->i.options & OPT_IMAP_SHAREDSEEN)
4162 repack->userid = xstrdup("anyone");
4163 else
4164 repack->userid = mboxname_to_userid(mailbox->name);
4165
4166 if (repack->userid) {
4167 struct seen *seendb = NULL;
4168 r = seen_open(repack->userid, SEEN_SILENT, &seendb);
4169 if (!r) r = seen_read(seendb, mailbox->uniqueid, &sd);
4170 seen_close(&seendb);
4171 }
4172
4173 if (!r) {
4174 repack->newmailbox.i.recentuid = sd.lastuid;
4175 repack->newmailbox.i.recenttime = sd.lastchange;
4176 repack->seqset = seqset_parse(sd.seenuids, NULL, sd.lastuid);
4177 seen_freedata(&sd);
4178 }
4179 }
4180 else if (version < 12 && mailbox->i.minor_version >= 12) {
4181 if (mailbox->i.options & OPT_IMAP_SHAREDSEEN)
4182 repack->userid = xstrdup("anyone");
4183 else
4184 repack->userid = mboxname_to_userid(mailbox->name);
4185
4186 /* we need to create the seen state for the owner from the mailbox */
4187 if (repack->userid)
4188 repack->seqset = seqset_init(mailbox->i.last_uid, SEQ_MERGE);
4189 }
4190
4191 /* we'll count the records as they get added */
4192 repack->newmailbox.i.num_records = 0;
4193 /* we're recreating caches, so there'll be nothing leaked */
4194 repack->newmailbox.i.leaked_cache_records = 0;
4195
4196 /* prepare initial header buffer */
4197 mailbox_index_header_to_buf(&repack->newmailbox.i, buf);
4198
4199 n = retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.start_offset);
4200 if (n == -1) goto fail;
4201
4202 if (version != mailbox->i.minor_version) {
4203 /* we're might be reading this file as we go, so let's
4204 * start the job by mapping the new file into the newmailbox!
4205 */
4206 repack->newmailbox.index_size = 0;
4207 repack->newmailbox.index_base = NULL;
4208 repack->newmailbox.index_len = 0;
4209 int r = mailbox_refresh_index_map(&repack->newmailbox);
4210 if (r) goto fail;
4211 }
4212
4213 *repackptr = repack;
4214 return 0;
4215
4216 fail:
4217 mailbox_repack_abort(&repack);
4218 return IMAP_IOERROR;
4219 }
4220
mailbox_repack_add(struct mailbox_repack * repack,struct index_record * record)4221 static int mailbox_repack_add(struct mailbox_repack *repack,
4222 struct index_record *record)
4223 {
4224 struct mappedfile *cachefile;
4225 indexbuffer_t ibuf;
4226 unsigned char *buf = ibuf.buf;
4227 int r;
4228 int n;
4229
4230 cachefile = repack_cachefile(repack, record);
4231
4232 /* write out the new cache record - need to clear the cache_offset
4233 * so it gets reset in the new record */
4234 record->cache_offset = 0;
4235 r = cache_append_record(cachefile, record);
4236 if (r) return r;
4237
4238 /* write the index record out */
4239 mailbox_index_record_to_buf(record, repack->newmailbox.i.minor_version, buf);
4240 n = retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.record_size);
4241 if (n == -1)
4242 return IMAP_IOERROR;
4243
4244 repack->newmailbox.i.num_records++;
4245
4246 return 0;
4247 }
4248
mailbox_repack_abort(struct mailbox_repack ** repackptr)4249 static void mailbox_repack_abort(struct mailbox_repack **repackptr)
4250 {
4251 struct mailbox_repack *repack = *repackptr;
4252 int i;
4253
4254 if (!repack) return; /* safe against double-free */
4255
4256 seqset_free(repack->seqset);
4257
4258 /* close and remove index */
4259 xclose(repack->newmailbox.index_fd);
4260 unlink(mailbox_meta_newfname(repack->mailbox, META_INDEX));
4261
4262 /* close and remove all new caches */
4263 for (i = 0; i < repack->caches.count; i++) {
4264 struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i);
4265 char *fname = xstrdup(mappedfile_fname(cachefile));
4266 mappedfile_commit(cachefile); /* gotta commit to clear the dirty flag. Alternative would be an unlink function */
4267 mappedfile_unlock(cachefile);
4268 mappedfile_close(&cachefile);
4269 unlink(fname);
4270 free(fname);
4271 }
4272 ptrarray_fini(&repack->caches);
4273
4274 // drop the map if we've mapped in the newmailbox index separately
4275 if (repack->newmailbox.index_base != repack->mailbox->index_base) {
4276 map_free(&repack->newmailbox.index_base, &repack->newmailbox.index_len);
4277 }
4278
4279 mailbox_abort(repack->mailbox);
4280
4281 free(repack->userid);
4282 free(repack);
4283 *repackptr = NULL;
4284
4285 return;
4286 }
4287
mailbox_repack_commit(struct mailbox_repack ** repackptr)4288 HIDDEN int mailbox_repack_commit(struct mailbox_repack **repackptr)
4289 {
4290 strarray_t cachefiles = STRARRAY_INITIALIZER;
4291 indexbuffer_t ibuf;
4292 unsigned char *buf = ibuf.buf;
4293 struct mailbox_repack *repack = *repackptr;
4294 int r = IMAP_IOERROR;
4295 int i;
4296
4297 assert(repack);
4298
4299 /* if we changed versions, we'll re-calculate counts on the new mailbox too */
4300 if (repack->newmailbox.i.minor_version != repack->mailbox->i.minor_version) {
4301 r = mailbox_refresh_index_map(&repack->newmailbox);
4302 if (r) goto fail;
4303 r = mailbox_index_recalc(&repack->newmailbox);
4304 if (r) goto fail;
4305 }
4306
4307 if (repack->newmailbox.i.minor_version >= 10 &&
4308 repack->mailbox->i.minor_version >= 10 &&
4309 !mailbox_crceq(repack->newmailbox.i.synccrcs, repack->crcs)) {
4310 syslog(LOG_ERR, "IOERROR: CRC mismatch on repack commit: %s (%u %u) (%u %u)",
4311 repack->mailbox->name,
4312 repack->crcs.basic, repack->newmailbox.i.synccrcs.basic,
4313 repack->crcs.annot, repack->newmailbox.i.synccrcs.annot);
4314 r = IMAP_MAILBOX_CHECKSUM;
4315 goto fail;
4316 }
4317
4318 repack->newmailbox.i.last_repack_time = time(0);
4319
4320 if (repack->mailbox->i.minor_version >= 12 && repack->newmailbox.i.minor_version < 12
4321 && repack->seqset && repack->userid) {
4322 struct seendata sd = SEENDATA_INITIALIZER;
4323 struct seen *seendb = NULL;
4324 int r = seen_open(repack->userid, SEEN_CREATE, &seendb);
4325 if (!r) r = seen_lockread(seendb, repack->mailbox->uniqueid, &sd);
4326 if (!r) {
4327 sd.lastuid = repack->newmailbox.i.last_uid;
4328 sd.seenuids = seqset_cstring(repack->seqset);
4329 if (!sd.seenuids) sd.seenuids = xstrdup("");
4330 sd.lastread = time(NULL);
4331 sd.lastchange = repack->newmailbox.i.last_appenddate;
4332 r = seen_write(seendb, repack->mailbox->uniqueid, &sd);
4333 /* XXX - syslog on errors? */
4334 }
4335 seen_close(&seendb);
4336 seen_freedata(&sd);
4337 }
4338
4339 /* rewrite the header with updated details */
4340 mailbox_index_header_to_buf(&repack->newmailbox.i, buf);
4341
4342 if (lseek(repack->newmailbox.index_fd, 0, SEEK_SET) < 0)
4343 goto fail;
4344
4345 if (retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.start_offset) < 0)
4346 goto fail;
4347
4348 /* ensure everything is committed to disk */
4349 if (fsync(repack->newmailbox.index_fd) < 0)
4350 goto fail;
4351
4352 xclose(repack->newmailbox.index_fd);
4353
4354 /* NOTE: cache files need committing before index is renamed */
4355 for (i = 0; i < repack->caches.count; i++) {
4356 struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i);
4357 r = mappedfile_commit(cachefile);
4358 if (r) goto fail;
4359 }
4360
4361 /* rename index first - loader will handle un-renamed cache if
4362 * the generation is lower */
4363 r = mailbox_meta_rename(repack->mailbox, META_INDEX);
4364 if (r) goto fail;
4365
4366 /* which cache files might currently exist? */
4367 strarray_add(&cachefiles, mailbox_meta_fname(repack->mailbox, META_CACHE));
4368 strarray_add(&cachefiles, mailbox_meta_fname(repack->mailbox, META_ARCHIVECACHE));
4369
4370 /* now the cache files can be renamed */
4371 for (i = 0; i < repack->caches.count; i++) {
4372 struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i);
4373 char *newname = xstrdup(mappedfile_fname(cachefile));
4374 size_t len = strlen(newname)-4;
4375 assert(!strcmp(newname+len, ".NEW"));
4376 newname[len] = '\0'; /* STRIP .NEW */
4377 mappedfile_rename(cachefile, newname);
4378 mappedfile_close(&cachefile);
4379 strarray_remove_all(&cachefiles, newname);
4380 free(newname);
4381 }
4382 ptrarray_fini(&repack->caches);
4383
4384 for (i = 0; i < cachefiles.count; i++) {
4385 const char *fname = strarray_nth(&cachefiles, i);
4386 if (!unlink(fname))
4387 syslog(LOG_NOTICE, "Removed unused cache file %s", fname);
4388 }
4389
4390 strarray_fini(&cachefiles);
4391
4392 // drop the map if we've mapped in the newmailbox index separately
4393 if (repack->newmailbox.index_base != repack->mailbox->index_base) {
4394 map_free(&repack->newmailbox.index_base, &repack->newmailbox.index_len);
4395 }
4396
4397 seqset_free(repack->seqset);
4398 free(repack->userid);
4399 free(repack);
4400 *repackptr = NULL;
4401 return 0;
4402
4403 fail:
4404 strarray_fini(&cachefiles);
4405 mailbox_repack_abort(repackptr);
4406 if (!r) r = IMAP_IOERROR;
4407 return r;
4408 }
4409
4410 /* need a mailbox exclusive lock, we're rewriting files */
mailbox_index_repack(struct mailbox * mailbox,int version)4411 static int mailbox_index_repack(struct mailbox *mailbox, int version)
4412 {
4413 struct mailbox_repack *repack = NULL;
4414 const message_t *msg;
4415 struct mailbox_iter *iter = NULL;
4416 struct buf buf = BUF_INITIALIZER;
4417 int r = IMAP_IOERROR;
4418
4419 syslog(LOG_INFO, "Repacking mailbox %s version %d", mailbox->name, version);
4420
4421 r = mailbox_repack_setup(mailbox, version, &repack);
4422 if (r) goto done;
4423
4424 iter = mailbox_iter_init(mailbox, 0, 0);
4425 while ((msg = mailbox_iter_step(iter))) {
4426 const struct index_record *record = msg_record(msg);
4427 struct index_record copyrecord = *record;
4428 int needs_cache_upgrade = 0;
4429 annotate_state_t *astate = NULL;
4430
4431 r = mailbox_get_annotate_state(mailbox, record->uid, &astate);
4432 if (r) goto done;
4433
4434 /* version changes? */
4435 if (mailbox->i.minor_version < 12 && repack->newmailbox.i.minor_version >= 12) {
4436 if (seqset_ismember(repack->seqset, copyrecord.uid))
4437 copyrecord.system_flags |= FLAG_SEEN;
4438 else
4439 copyrecord.system_flags &= ~FLAG_SEEN;
4440
4441 needs_cache_upgrade = 1;
4442
4443 }
4444 if (mailbox->i.minor_version >= 12 && repack->newmailbox.i.minor_version < 12) {
4445 if (repack->seqset)
4446 seqset_add(repack->seqset, copyrecord.uid, copyrecord.system_flags & FLAG_SEEN ? 1 : 0);
4447 copyrecord.system_flags &= ~FLAG_SEEN;
4448 }
4449
4450 /* force cache upgrade across version 15 repack */
4451 if (repack->newmailbox.i.minor_version >= 15 && record->cache_version < 9)
4452 needs_cache_upgrade = 1;
4453
4454 /* better handle the cleanup just in case it's unlinked too */
4455 /* still gotta check for FLAG_INTERNAL_UNLINKED, because it may have been
4456 * created by old code. Woot */
4457 if (copyrecord.internal_flags & (FLAG_INTERNAL_NEEDS_CLEANUP | FLAG_INTERNAL_UNLINKED)) {
4458 mailbox_record_cleanup(mailbox, ©record);
4459 copyrecord.internal_flags &= ~FLAG_INTERNAL_NEEDS_CLEANUP;
4460 /* no need to rewrite - it's already being written to the new file */
4461 }
4462
4463 /* we aren't keeping unlinked files, that's kind of the point */
4464 if (copyrecord.internal_flags & FLAG_INTERNAL_UNLINKED) {
4465 /* track the modseq for QRESYNC purposes */
4466 if (copyrecord.modseq > repack->newmailbox.i.deletedmodseq)
4467 repack->newmailbox.i.deletedmodseq = copyrecord.modseq;
4468 continue;
4469 }
4470
4471 if (needs_cache_upgrade) {
4472 const char *fname = mailbox_record_fname(mailbox, ©record);
4473
4474 if (message_parse(fname, ©record)) {
4475 /* failed to parse, don't try to write out record */
4476 copyrecord.crec.len = 0;
4477 /* and the record is expunged too! */
4478 copyrecord.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
4479 syslog(LOG_ERR, "IOERROR: FATAL - failed to parse file for %s %u, expunging",
4480 repack->mailbox->name, copyrecord.uid);
4481 }
4482 }
4483
4484 /* virtual annotations */
4485 if (mailbox->i.minor_version < 13 && repack->newmailbox.i.minor_version >= 13) {
4486 /* extract CID */
4487 buf_reset(&buf);
4488 mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "thrid", "", &buf);
4489 if (buf.len == 16) {
4490 const char *p = buf_cstring(&buf);
4491 parsehex(p, &p, 16, ©record.cid);
4492 }
4493 buf_reset(&buf);
4494 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "thrid", "", &buf);
4495 if (r) goto done;
4496 }
4497 if (mailbox->i.minor_version >= 13 && repack->newmailbox.i.minor_version < 13) {
4498 if (record->cid) {
4499 buf_reset(&buf);
4500 buf_printf(&buf, "%llx", record->cid);
4501 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "thrid", "", &buf);
4502 if (r) goto done;
4503 }
4504 }
4505
4506 if (mailbox->i.minor_version < 15 && repack->newmailbox.i.minor_version >= 15) {
4507 /* extract CID */
4508 buf_reset(&buf);
4509 mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "savedate", "", &buf);
4510 if (buf.len) {
4511 const char *p = buf_cstring(&buf);
4512 bit64 newval;
4513 parsenum(p, &p, 0, &newval);
4514 copyrecord.savedate = newval;
4515 }
4516 buf_reset(&buf);
4517 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "savedate", "", &buf);
4518 if (r) goto done;
4519 }
4520 if (mailbox->i.minor_version >= 15 && repack->newmailbox.i.minor_version < 15) {
4521 if (record->savedate) {
4522 buf_reset(&buf);
4523 buf_printf(&buf, TIME_T_FMT, record->savedate);
4524 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "savedate", "", &buf);
4525 if (r) goto done;
4526 }
4527 }
4528
4529 if (mailbox->i.minor_version < 16 && repack->newmailbox.i.minor_version >= 16) {
4530 /* extract CID */
4531 buf_reset(&buf);
4532 mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf);
4533 if (buf.len) {
4534 const char *p = buf_cstring(&buf);
4535 bit64 newval;
4536 parsenum(p, &p, 0, &newval);
4537 copyrecord.createdmodseq = newval;
4538 }
4539 buf_reset(&buf);
4540 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "createdmodseq", "", &buf);
4541 if (r) goto done;
4542 }
4543 if (mailbox->i.minor_version >= 16 && repack->newmailbox.i.minor_version < 16) {
4544 if (record->createdmodseq) {
4545 buf_reset(&buf);
4546 buf_printf(&buf, "%llu", record->createdmodseq);
4547 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "createdmodseq", "", &buf);
4548 if (r) goto done;
4549 }
4550 }
4551
4552 /* read in the old cache record */
4553 r = mailbox_cacherecord(mailbox, ©record);
4554 if (r) goto done;
4555
4556 r = mailbox_repack_add(repack, ©record);
4557 if (r) goto done;
4558 }
4559
4560 /* we unlinked any "needs unlink" in the process */
4561 repack->newmailbox.i.options &= ~(OPT_MAILBOX_NEEDS_REPACK|OPT_MAILBOX_NEEDS_UNLINK);
4562
4563 done:
4564 mailbox_iter_done(&iter);
4565 buf_free(&buf);
4566 if (r) mailbox_repack_abort(&repack);
4567 else r = mailbox_repack_commit(&repack);
4568 return r;
4569 }
4570
4571 /*
4572 * Used by mailbox_rename() to expunge all messages in INBOX
4573 */
expungeall(struct mailbox * mailbox,const struct index_record * record,void * rock)4574 static unsigned expungeall(struct mailbox *mailbox __attribute__((unused)),
4575 const struct index_record *record __attribute__((unused)),
4576 void *rock __attribute__((unused)))
4577 {
4578 return 1;
4579 }
4580
4581 /*
4582 * Expunge decision proc used by mailbox_expunge()
4583 * to expunge \Deleted messages.
4584 */
expungedeleted(struct mailbox * mailbox,const struct index_record * record,void * rock)4585 static unsigned expungedeleted(struct mailbox *mailbox __attribute__((unused)),
4586 const struct index_record *record,
4587 void *rock __attribute__((unused)))
4588 {
4589 if (record->system_flags & FLAG_DELETED)
4590 return 1;
4591
4592 return 0;
4593 }
4594
mailbox_should_archive(struct mailbox * mailbox,const struct index_record * record,void * rock)4595 EXPORTED unsigned mailbox_should_archive(struct mailbox *mailbox,
4596 const struct index_record *record,
4597 void *rock)
4598 {
4599 int archive_after = config_getduration(IMAPOPT_ARCHIVE_AFTER, 'd');
4600 time_t cutoff = time(0) - archive_after;
4601 if (rock) cutoff = *((time_t *)rock);
4602
4603 int archive_size = config_getint(IMAPOPT_ARCHIVE_MAXSIZE);
4604 size_t maxsize = archive_size * 1024;
4605
4606 int keepflagged = config_getswitch(IMAPOPT_ARCHIVE_KEEPFLAGGED);
4607
4608 /* never pull messages back from the archives */
4609 if (record->internal_flags & FLAG_INTERNAL_ARCHIVED)
4610 return 1;
4611
4612 /* first check if we're archiving anything */
4613 if (!config_getswitch(IMAPOPT_ARCHIVE_ENABLED))
4614 return 0;
4615
4616 /* always archive big messages */
4617 if (record->size >= maxsize)
4618 return 1;
4619
4620 /* archive everything in DELETED mailboxes */
4621 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
4622 return 1;
4623
4624 /* anything already deleted */
4625 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED)
4626 return 1;
4627
4628 /* Calendar and Addressbook are small files and need to be hot */
4629 if (mailbox->mbtype & MBTYPE_ADDRESSBOOK)
4630 return 0;
4631 if (mailbox->mbtype & MBTYPE_CALENDAR)
4632 return 0;
4633
4634 /* don't archive flagged messages */
4635 if (keepflagged && (record->system_flags & FLAG_FLAGGED))
4636 return 0;
4637
4638 /* archive all other old messages */
4639 if (record->internaldate <= cutoff)
4640 return 1;
4641
4642 /* and don't archive anything else! */
4643 return 0;
4644 }
4645
4646 /*
4647 * Move messages between spool and archive partition
4648 * function pointed to by 'decideproc' is called (with 'deciderock') to
4649 * determine which messages to move. If deciderock return 0, the message
4650 * should be in the spool - if 1, the message should be in the archive.
4651 */
mailbox_archive(struct mailbox * mailbox,mailbox_decideproc_t * decideproc,void * deciderock,unsigned flags)4652 EXPORTED void mailbox_archive(struct mailbox *mailbox,
4653 mailbox_decideproc_t *decideproc, void *deciderock, unsigned flags)
4654
4655 {
4656 int r;
4657 int dirtycache = 0;
4658 const message_t *msg;
4659 struct index_record copyrecord;
4660 const char *srcname;
4661 const char *destname;
4662 char *spoolcache = xstrdup(mailbox_meta_fname(mailbox, META_CACHE));
4663 char *archivecache = xstrdup(mailbox_meta_fname(mailbox, META_ARCHIVECACHE));
4664 int differentcache = strcmp(spoolcache, archivecache);
4665 int object_storage_enabled = 0;
4666 #if defined ENABLE_OBJECTSTORE
4667 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
4668 #endif
4669 free(spoolcache);
4670 free(archivecache);
4671
4672 assert(mailbox_index_islocked(mailbox, 1));
4673 if (!decideproc) decideproc = &mailbox_should_archive;
4674
4675 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags);
4676
4677 while ((msg = mailbox_iter_step(iter))) {
4678 const struct index_record *record = msg_record(msg);
4679 const char *action = NULL;
4680 if (decideproc(mailbox, record, deciderock)) {
4681 if (record->internal_flags & FLAG_INTERNAL_ARCHIVED)
4682 continue;
4683 copyrecord = *record;
4684 srcname = mailbox_spool_fname(mailbox, copyrecord.uid);
4685 destname = mailbox_archive_fname(mailbox, copyrecord.uid);
4686
4687 /* load cache before changing the flags */
4688 r = mailbox_cacherecord(mailbox, ©record);
4689 if (r) {
4690 syslog(LOG_ERR, "IOERROR archive %s %u failed to read cache: %s",
4691 mailbox->name, copyrecord.uid, error_message(r));
4692 continue;
4693 }
4694 #if defined ENABLE_OBJECTSTORE
4695 if (object_storage_enabled){
4696 /* upload on the blob store */
4697 r = objectstore_put(mailbox, ©record, srcname);
4698 if (r) {
4699 syslog(LOG_ERR, "IOERROR archive %s %u failed to objectstorage put file (%s): %s",
4700 mailbox->name, copyrecord.uid, srcname, error_message(r));
4701 // didn't manage to store it, so remove the ARCHIVED flag
4702 continue;
4703 }
4704 r = unlink (srcname);
4705 if (r < 0)
4706 syslog(LOG_ERR, "unlink(%s) failed: %m", srcname);
4707 }
4708 #endif
4709 copyrecord.internal_flags |= FLAG_INTERNAL_ARCHIVED | FLAG_INTERNAL_NEEDS_CLEANUP;
4710 action = "archive";
4711 }
4712 else
4713 {
4714 if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED))
4715 continue;
4716 copyrecord = *record;
4717 destname = mailbox_spool_fname(mailbox, copyrecord.uid);
4718 srcname = mailbox_archive_fname(mailbox, copyrecord.uid);
4719
4720 /* load cache before changing the flags */
4721 r = mailbox_cacherecord(mailbox, ©record);
4722 if (r) {
4723 syslog(LOG_ERR, "IOERROR archive %s %u failed to read cache: %s",
4724 mailbox->name, copyrecord.uid, error_message(r));
4725 continue;
4726 }
4727
4728 #if defined ENABLE_OBJECTSTORE
4729 if (object_storage_enabled){
4730 /* recover from the blob store */
4731 r = objectstore_get(mailbox, ©record, destname);
4732 if (r) {
4733 syslog(LOG_ERR, "IOERROR archive %s %u failed to objectstorage get file (%s): %s",
4734 mailbox->name, copyrecord.uid, destname, error_message(r));
4735 continue;
4736 }
4737 objectstore_delete(mailbox, ©record); // this should only lower ref count.
4738 }
4739 #endif
4740
4741 copyrecord.internal_flags &= ~FLAG_INTERNAL_ARCHIVED;
4742 copyrecord.internal_flags |= FLAG_INTERNAL_NEEDS_CLEANUP;
4743 action = "unarchive";
4744 }
4745
4746 if (!object_storage_enabled){
4747 /* got a file to copy! */
4748 if (strcmp(srcname, destname)) {
4749 r = cyrus_copyfile(srcname, destname, COPYFILE_MKDIR|COPYFILE_KEEPTIME);
4750 if (r) {
4751 syslog(LOG_ERR, "IOERROR archive %s %u failed to copyfile (%s => %s): %s",
4752 mailbox->name, copyrecord.uid, srcname, destname, error_message(r));
4753 continue;
4754 }
4755 }
4756 }
4757
4758 /* got a new cache record to write */
4759 if (differentcache)
4760 {
4761 dirtycache = 1;
4762 copyrecord.cache_offset = 0;
4763 if (mailbox_append_cache(mailbox, ©record))
4764 continue;
4765 }
4766
4767 /* rewrite the index record */
4768 copyrecord.silent = 1;
4769 if (mailbox_rewrite_index_record(mailbox, ©record))
4770 continue;
4771 mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK;
4772
4773 if (config_auditlog) {
4774 char flagstr[FLAGMAPSTR_MAXLEN];
4775 flags_to_str(©record, flagstr);
4776 syslog(LOG_NOTICE, "auditlog: %s sessionid=<%s> mailbox=<%s> "
4777 "uniqueid=<%s> uid=<%u> guid=<%s> cid=<%s> sysflags=<%s>",
4778 action, session_id(), mailbox->name, mailbox->uniqueid,
4779 copyrecord.uid, message_guid_encode(©record.guid),
4780 conversation_id_encode(copyrecord.cid), flagstr);
4781 }
4782 }
4783 mailbox_iter_done(&iter);
4784
4785 /* if we have stale cache records, we'll need a repack */
4786 if (dirtycache) {
4787 mailbox_index_dirty(mailbox);
4788 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
4789 }
4790 }
4791
mailbox_remove_files_from_object_storage(struct mailbox * mailbox,unsigned flags)4792 EXPORTED void mailbox_remove_files_from_object_storage(struct mailbox *mailbox,
4793 unsigned flags)
4794
4795 {
4796 const message_t *msg;
4797 #if defined ENABLE_OBJECTSTORE
4798 int object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
4799 #endif
4800
4801 assert(mailbox_index_islocked(mailbox, 1));
4802 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags);
4803 while ((msg = mailbox_iter_step(iter))) {
4804 const struct index_record *record = msg_record(msg);
4805 if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED))
4806 continue;
4807 #if defined ENABLE_OBJECTSTORE
4808 if (object_storage_enabled)
4809 objectstore_delete(mailbox, record); // this should only lower ref count.
4810 #endif
4811 }
4812 mailbox_iter_done(&iter);
4813 }
4814
4815
4816 /*
4817 * Perform an expunge operation on 'mailbox'. If nonzero, the
4818 * function pointed to by 'decideproc' is called (with 'deciderock') to
4819 * determine which messages to expunge. If 'decideproc' is a null pointer,
4820 * then messages with the \Deleted flag are expunged.
4821 *
4822 * event_type - the event among MessageExpunge, MessageExpire (zero means
4823 * don't send notification)
4824 */
mailbox_expunge(struct mailbox * mailbox,mailbox_decideproc_t * decideproc,void * deciderock,unsigned * nexpunged,int event_type)4825 EXPORTED int mailbox_expunge(struct mailbox *mailbox,
4826 mailbox_decideproc_t *decideproc, void *deciderock,
4827 unsigned *nexpunged, int event_type)
4828 {
4829 int r = 0;
4830 int numexpunged = 0;
4831 const message_t *msg;
4832 struct mboxevent *mboxevent = NULL;
4833
4834 assert(mailbox_index_islocked(mailbox, 1));
4835
4836 /* anything to do? */
4837 if (!mailbox->i.num_records) {
4838 if (nexpunged) *nexpunged = 0;
4839 return 0;
4840 }
4841
4842 if (event_type)
4843 mboxevent = mboxevent_new(event_type);
4844
4845 if (!decideproc) decideproc = expungedeleted;
4846
4847 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED);
4848 while ((msg = mailbox_iter_step(iter))) {
4849 const struct index_record *record = msg_record(msg);
4850 if (decideproc(mailbox, record, deciderock)) {
4851 numexpunged++;
4852
4853 struct index_record copyrecord = *record;
4854 /* mark deleted */
4855 copyrecord.internal_flags |= FLAG_INTERNAL_EXPUNGED;
4856
4857 r = mailbox_rewrite_index_record(mailbox, ©record);
4858 if (r) {
4859 mboxevent_free(&mboxevent);
4860 mailbox_iter_done(&iter);
4861 return IMAP_IOERROR;
4862 }
4863
4864 mboxevent_extract_record(mboxevent, mailbox, ©record);
4865 }
4866 }
4867 mailbox_iter_done(&iter);
4868
4869 if (numexpunged > 0) {
4870 syslog(LOG_NOTICE, "Expunged %d messages from %s",
4871 numexpunged, mailbox->name);
4872
4873 /* send the MessageExpunge or MessageExpire event notification */
4874 mboxevent_extract_mailbox(mboxevent, mailbox);
4875 mboxevent_set_access(mboxevent, NULL, NULL, "", mailbox->name, 0);
4876 mboxevent_set_numunseen(mboxevent, mailbox, -1);
4877 mboxevent_notify(&mboxevent);
4878 }
4879 mboxevent_free(&mboxevent);
4880
4881 if (nexpunged) *nexpunged = numexpunged;
4882
4883 return 0;
4884 }
4885
mailbox_expunge_cleanup(struct mailbox * mailbox,time_t expunge_mark,unsigned * ndeleted)4886 EXPORTED int mailbox_expunge_cleanup(struct mailbox *mailbox, time_t expunge_mark,
4887 unsigned *ndeleted)
4888 {
4889 int dirty = 0;
4890 unsigned numdeleted = 0;
4891 const message_t *msg;
4892 time_t first_expunged = 0;
4893 int r = 0;
4894
4895 /* run the actual expunge phase */
4896 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, 0);
4897 while ((msg = mailbox_iter_step(iter))) {
4898 const struct index_record *record = msg_record(msg);
4899 /* already unlinked, skip it (but dirty so we mark a repack is needed) */
4900 if (record->internal_flags & FLAG_INTERNAL_UNLINKED) {
4901 dirty = 1;
4902 continue;
4903 }
4904
4905 /* not actually expunged, skip it */
4906 if (!(record->internal_flags & FLAG_INTERNAL_EXPUNGED))
4907 continue;
4908
4909 /* not stale enough yet, skip it - but track the updated time
4910 * so we know when to run again */
4911 if (record->last_updated > expunge_mark) {
4912 if (!first_expunged || (first_expunged > record->last_updated))
4913 first_expunged = record->last_updated;
4914 continue;
4915 }
4916
4917 dirty = 1;
4918
4919 numdeleted++;
4920
4921 struct index_record copyrecord = *record;
4922 copyrecord.internal_flags |= FLAG_INTERNAL_UNLINKED;
4923 copyrecord.silent = 1;
4924 if (mailbox_rewrite_index_record(mailbox, ©record)) {
4925 syslog(LOG_ERR, "IOERROR: failed to mark unlinked %s %u (recno %d)",
4926 mailbox->name, copyrecord.uid, copyrecord.recno);
4927 break;
4928 }
4929 }
4930 mailbox_iter_done(&iter);
4931
4932 if (dirty) {
4933 mailbox_index_dirty(mailbox);
4934 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
4935 mailbox->i.first_expunged = first_expunged;
4936 }
4937
4938 if (ndeleted) *ndeleted = numdeleted;
4939
4940 return r;
4941 }
4942
mailbox_internal_seen(const struct mailbox * mailbox,const char * userid)4943 EXPORTED int mailbox_internal_seen(const struct mailbox *mailbox, const char *userid)
4944 {
4945 /* old mailboxes don't have internal seen at all */
4946 if (mailbox->i.minor_version < 12)
4947 return 0;
4948
4949 /* shared seen - everyone's state is internal */
4950 if (mailbox->i.options & OPT_IMAP_SHAREDSEEN)
4951 return 1;
4952
4953 /* no username => use internal as well */
4954 if (!userid)
4955 return 1;
4956
4957 /* otherwise the owner's seen state is internal */
4958 return mboxname_userownsmailbox(userid, mailbox->name);
4959 }
4960
4961 /*
4962 * Return the number of message without \Seen flag in a mailbox.
4963 * Suppose that authenticated user is the owner or sharedseen is enabled
4964 */
mailbox_count_unseen(struct mailbox * mailbox)4965 unsigned mailbox_count_unseen(struct mailbox *mailbox)
4966 {
4967 assert(mailbox_index_islocked(mailbox, 0));
4968
4969 if (mailbox->i.minor_version > 13)
4970 return mailbox->i.unseen;
4971
4972 const message_t *msg;
4973 unsigned count = 0;
4974
4975 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED);
4976 while ((msg = mailbox_iter_step(iter))) {
4977 const struct index_record *record = msg_record(msg);
4978 if (!(record->system_flags & FLAG_SEEN))
4979 count++;
4980 }
4981 mailbox_iter_done(&iter);
4982
4983 return count;
4984 }
4985
4986 /* returns a mailbox locked in MAILBOX EXCLUSIVE mode, so you
4987 * don't need to lock the index file to work with it :) */
mailbox_create(const char * name,uint32_t mbtype,const char * part,const char * acl,const char * uniqueid,int options,unsigned uidvalidity,modseq_t createdmodseq,modseq_t highestmodseq,struct mailbox ** mailboxptr)4988 EXPORTED int mailbox_create(const char *name,
4989 uint32_t mbtype,
4990 const char *part,
4991 const char *acl,
4992 const char *uniqueid,
4993 int options,
4994 unsigned uidvalidity,
4995 modseq_t createdmodseq,
4996 modseq_t highestmodseq,
4997 struct mailbox **mailboxptr)
4998 {
4999 int r = 0;
5000 char quotaroot[MAX_MAILBOX_BUFFER];
5001 int hasquota;
5002 const char *fname;
5003 struct mailbox *mailbox = NULL;
5004 int n;
5005 int createfnames[] = { META_INDEX, META_HEADER, 0 };
5006 struct mailboxlist *listitem;
5007 strarray_t *initial_flags = NULL;
5008
5009 if (!uniqueid) uniqueid = makeuuid();
5010
5011 /* if we already have this name open then that's an error too */
5012 listitem = find_listitem(name);
5013 if (listitem) return IMAP_MAILBOX_LOCKED;
5014
5015 listitem = create_listitem(name);
5016 mailbox = &listitem->m;
5017
5018 /* if we can't get an exclusive lock first try, there's something
5019 * racy going on! */
5020 /* an exclusive lock around the non-exclusive lock to avoid create
5021 * races on fixed names like calendar Inbox and Outbox */
5022 struct mboxlock *mblock = NULL;
5023 r = mboxname_lock("$CREATE", &mblock, LOCK_EXCLUSIVE);
5024 if (!r) r = mboxname_lock(name, &listitem->l, LOCK_NONBLOCKING);
5025 mboxname_release(&mblock);
5026 if (r) goto done;
5027
5028 mailbox->part = xstrdup(part);
5029 mailbox->acl = xstrdup(acl);
5030 mailbox->mbtype = mbtype;
5031 mailbox->uniqueid = xstrdup(uniqueid);
5032 // if we've been given a highestmodseq, we don't update it
5033 if (highestmodseq) mailbox->silentchanges = 1;
5034
5035 hasquota = quota_findroot(quotaroot, sizeof(quotaroot), name);
5036
5037 /* ensure all paths exist */
5038 for (n = 0; createfnames[n]; n++) {
5039 fname = mailbox_meta_fname(mailbox, createfnames[n]);
5040 if (!fname) {
5041 syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
5042 r = IMAP_MAILBOX_BADNAME;
5043 goto done;
5044 }
5045 if (cyrus_mkdir(fname, 0755) == -1) {
5046 syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
5047 r = IMAP_IOERROR;
5048 goto done;
5049 }
5050 }
5051
5052 /* ensure we can fit the longest possible file name */
5053 fname = mailbox_datapath(mailbox, 0);
5054 if (!fname) {
5055 syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
5056 r = IMAP_MAILBOX_BADNAME;
5057 goto done;
5058 }
5059 /* and create the directory too :) */
5060 if (cyrus_mkdir(fname, 0755) == -1) {
5061 syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
5062 r = IMAP_IOERROR;
5063 goto done;
5064 }
5065
5066 /* open conversations FIRST */
5067 r = mailbox_lock_conversations(mailbox, LOCK_EXCLUSIVE);
5068 if (r) {
5069 syslog(LOG_ERR, "IOERROR: locking conversations %s %s",
5070 mailbox->name, error_message(r));
5071 r = IMAP_IOERROR;
5072 goto done;
5073 }
5074
5075 fname = mailbox_meta_fname(mailbox, META_INDEX);
5076 if (!fname) {
5077 syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
5078 r = IMAP_MAILBOX_BADNAME;
5079 goto done;
5080 }
5081 mailbox->index_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
5082 if (mailbox->index_fd == -1) {
5083 syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
5084 r = IMAP_IOERROR;
5085 goto done;
5086 }
5087 r = lock_blocking(mailbox->index_fd, fname);
5088 if (r) {
5089 syslog(LOG_ERR, "IOERROR: locking %s: %m", fname);
5090 r = IMAP_IOERROR;
5091 goto done;
5092 }
5093 mailbox->index_locktype = LOCK_EXCLUSIVE;
5094
5095 if (hasquota) {
5096 mailbox_set_quotaroot(mailbox, quotaroot);
5097 memset(mailbox->quota_previously_used, 0, sizeof(mailbox->quota_previously_used));
5098 mailbox->quota_dirty = 1;
5099 }
5100
5101 /* ensure a UIDVALIDITY is set */
5102 if (!uidvalidity)
5103 uidvalidity = mboxname_nextuidvalidity(name, time(0));
5104 else
5105 mboxname_setuidvalidity(mailbox->name, uidvalidity);
5106
5107 /* and highest modseq */
5108 if (!highestmodseq)
5109 highestmodseq = mboxname_nextmodseq(mailbox->name, 0, mbtype, /*dofolder*/1);
5110 else
5111 mboxname_setmodseq(mailbox->name, highestmodseq, mbtype, /*dofolder*/1);
5112
5113 /* and created modseq */
5114 if (!createdmodseq || createdmodseq > highestmodseq)
5115 createdmodseq = highestmodseq;
5116
5117 /* init non-zero fields */
5118 mailbox_index_dirty(mailbox);
5119 mailbox->i.minor_version = MAILBOX_MINOR_VERSION;
5120 mailbox->i.start_offset = INDEX_HEADER_SIZE;
5121 mailbox->i.record_size = INDEX_RECORD_SIZE;
5122 mailbox->i.options = options;
5123 mailbox->i.uidvalidity = uidvalidity;
5124 mailbox->i.createdmodseq = createdmodseq;
5125 mailbox->i.highestmodseq = highestmodseq;
5126 mailbox->i.synccrcs.basic = CRC_INIT_BASIC;
5127 mailbox->i.synccrcs.annot = CRC_INIT_ANNOT;
5128
5129 /* initialise header size field so appends calculate the
5130 * correct map size */
5131 mailbox->index_size = INDEX_HEADER_SIZE;
5132
5133 mailbox->header_dirty = 1;
5134
5135 /* pre-set any required permanent flags */
5136 if (config_getstring(IMAPOPT_MAILBOX_INITIAL_FLAGS)) {
5137 const char *val = config_getstring(IMAPOPT_MAILBOX_INITIAL_FLAGS);
5138 int i;
5139
5140 initial_flags = strarray_split(val, NULL, 0);
5141
5142 for (i = 0; i < initial_flags->count; i++) {
5143 const char *flag = strarray_nth(initial_flags, i);
5144 r = mailbox_user_flag(mailbox, flag, NULL, /*create*/1);
5145 if (r) goto done;
5146 }
5147 }
5148
5149 r = seen_create_mailbox(NULL, mailbox);
5150 if (r) goto done;
5151 r = mailbox_commit(mailbox);
5152 if (r) goto done;
5153
5154 if (config_auditlog)
5155 syslog(LOG_NOTICE, "auditlog: create sessionid=<%s> "
5156 "mailbox=<%s> uniqueid=<%s> uidvalidity=<%u>",
5157 session_id(), mailbox->name,
5158 mailbox->uniqueid, mailbox->i.uidvalidity);
5159
5160 done:
5161 if (!r && mailboxptr)
5162 *mailboxptr = mailbox;
5163 else
5164 mailbox_close(&mailbox);
5165
5166 strarray_free(initial_flags);
5167
5168 return r;
5169 }
5170
5171 /*
5172 * Remove all files in directory
5173 */
mailbox_delete_files(const char * path)5174 static void mailbox_delete_files(const char *path)
5175 {
5176 DIR *dirp;
5177 struct dirent *f;
5178 char buf[MAX_MAILBOX_PATH+1];
5179 char *tail;
5180
5181 strlcpy(buf, path, sizeof(buf));
5182
5183 if(strlen(buf) >= sizeof(buf) - 2) {
5184 syslog(LOG_ERR, "IOERROR: Path too long (%s)", buf);
5185 fatal("path too long", EX_OSFILE);
5186 }
5187
5188 tail = buf + strlen(buf);
5189 *tail++ = '/';
5190 *tail = '\0';
5191 dirp = opendir(path);
5192 if (dirp) {
5193 while ((f = readdir(dirp))!=NULL) {
5194 if (f->d_name[0] == '.'
5195 && (f->d_name[1] == '\0'
5196 || (f->d_name[1] == '.' &&
5197 f->d_name[2] == '\0'))) {
5198 /* readdir() can return "." or "..", and I got a bug report
5199 that SCO might blow the file system to smithereens if we
5200 unlink(".."). Let's not do that. */
5201 continue;
5202 }
5203
5204 if(strlen(buf) + strlen(f->d_name) >= sizeof(buf)) {
5205 syslog(LOG_ERR, "IOERROR: Path too long (%s + %s)",
5206 buf, f->d_name);
5207 fatal("Path too long", EX_OSFILE);
5208 }
5209 strcpy(tail, f->d_name);
5210 unlink(buf);
5211 *tail = '\0';
5212 }
5213 closedir(dirp);
5214 }
5215 }
5216
5217 /* Callback for use by cmd_delete */
chkchildren(const mbentry_t * mbentry,void * rock)5218 static int chkchildren(const mbentry_t *mbentry,
5219 void *rock)
5220 {
5221 const char *part = (const char *)rock;
5222
5223 if (!strcmpnull(part, mbentry->partition))
5224 return CYRUSDB_DONE;
5225
5226 return 0;
5227 }
5228
5229 #ifdef WITH_DAV
mailbox_add_dav(struct mailbox * mailbox)5230 EXPORTED int mailbox_add_dav(struct mailbox *mailbox)
5231 {
5232 const message_t *msg;
5233 int r = 0;
5234
5235 if (!(mailbox->mbtype & (MBTYPES_DAV)))
5236 return 0;
5237
5238 if (mboxname_isdeletedmailbox(mailbox->name, NULL))
5239 return 0;
5240
5241 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
5242 while ((msg = mailbox_iter_step(iter))) {
5243 const struct index_record *record = msg_record(msg);
5244 struct index_record copyrecord = *record;
5245 r = mailbox_update_dav(mailbox, NULL, ©record);
5246 if (r) break;
5247 /* in THEORY there maybe changes here that we should be saving... */
5248 }
5249 mailbox_iter_done(&iter);
5250
5251 return r;
5252 }
5253
5254 #endif /* WITH_DAV */
5255
mailbox_add_conversations(struct mailbox * mailbox,int silent)5256 EXPORTED int mailbox_add_conversations(struct mailbox *mailbox, int silent)
5257 {
5258 const message_t *msg;
5259 int r = 0;
5260
5261 struct conversations_state *cstate = mailbox_get_cstate(mailbox);
5262
5263 if (!cstate) return 0;
5264
5265 /* add record for mailbox */
5266 conv_status_t status = CONV_STATUS_INIT;
5267 status.threadmodseq = mailbox->i.highestmodseq;
5268 r = conversation_setstatus(cstate, mailbox->name, &status);
5269 if (r) return r;
5270
5271 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
5272 while ((msg = mailbox_iter_step(iter))) {
5273 const struct index_record *record = msg_record(msg);
5274 /* not assigned, skip */
5275 if (!record->cid)
5276 continue;
5277
5278 struct index_record copyrecord = *record;
5279 copyrecord.silent = silent;
5280 r = conversations_update_record(cstate, mailbox, NULL, ©record, 1);
5281 if (r) break;
5282
5283 if (copyrecord.cid == record->cid)
5284 continue;
5285
5286 assert(!silent); // can't change cid if silent!
5287
5288 /* remove this record again */
5289 r = conversations_update_record(cstate, mailbox, ©record, NULL, 0);
5290 if (r) break;
5291
5292 /* we had a cid change, so rewrite will try to correct the counts, so we
5293 * need to add this one in again */
5294 struct index_record oldrecord = *record;
5295 /* add the old record that's going away */
5296 r = conversations_update_record(cstate, mailbox, NULL, &oldrecord, 0);
5297 if (r) break;
5298
5299 /* and finally to the update that will reverse those two actions again */
5300 r = mailbox_rewrite_index_record(mailbox, ©record);
5301 if (r) break;
5302 }
5303 mailbox_iter_done(&iter);
5304
5305 return r;
5306 }
5307
mailbox_delete_conversations(struct mailbox * mailbox)5308 static int mailbox_delete_conversations(struct mailbox *mailbox)
5309 {
5310 struct conversations_state *cstate = mailbox_get_cstate(mailbox);
5311 const message_t *msg;
5312 int r = 0;
5313
5314 if (!cstate) return 0;
5315
5316 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
5317 while ((msg = mailbox_iter_step(iter))) {
5318 const struct index_record *record = msg_record(msg);
5319 /* not assigned, skip */
5320 if (!record->cid)
5321 continue;
5322
5323 r = conversations_update_record(cstate, mailbox, record, NULL, 0);
5324 if (r) break;
5325 }
5326 mailbox_iter_done(&iter);
5327 if (r) return r;
5328
5329 return conversations_rename_folder(cstate, mailbox->name, NULL);
5330 }
5331
mailbox_delete_internal(struct mailbox ** mailboxptr)5332 static int mailbox_delete_internal(struct mailbox **mailboxptr)
5333 {
5334 int r = 0;
5335 struct mailbox *mailbox = *mailboxptr;
5336
5337 /* mark the quota removed */
5338 mailbox_quota_dirty(mailbox);
5339
5340 /* mark the mailbox deleted */
5341 mailbox_index_dirty(mailbox);
5342 mailbox->i.options |= OPT_MAILBOX_DELETED;
5343
5344 #ifdef WITH_DAV
5345 /* remove any DAV records */
5346 r = mailbox_delete_dav(mailbox);
5347 if (r) return r;
5348 #endif
5349
5350 /* clean up annotations */
5351 r = annotate_delete_mailbox(mailbox);
5352 if (r) return r;
5353
5354 /* commit the changes */
5355 r = mailbox_commit(mailbox);
5356 if (r) return r;
5357
5358 /* remove any seen */
5359 seen_delete_mailbox(NULL, mailbox);
5360
5361 /* can't unlink any files yet, because our promise to other
5362 * users of the mailbox applies! Can only unlink with an
5363 * exclusive lock. mailbox_close will try to get one of
5364 * those.
5365 */
5366
5367 syslog(LOG_NOTICE, "Deleted mailbox %s", mailbox->name);
5368
5369 if (config_auditlog)
5370 syslog(LOG_NOTICE, "auditlog: delete sessionid=<%s> "
5371 "mailbox=<%s> uniqueid=<%s>",
5372 session_id(),
5373 mailbox->name, mailbox->uniqueid);
5374
5375 proc_killmbox(mailbox->name);
5376
5377 mailbox_close(mailboxptr);
5378
5379 return 0;
5380 }
5381
5382 #ifdef WITH_JMAP
mailbox_delete_alarms(struct mailbox * mailbox)5383 static int mailbox_delete_alarms(struct mailbox *mailbox)
5384 {
5385 if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS))
5386 return 0;
5387
5388 return caldav_alarm_delete_mailbox(mailbox->name);
5389 }
5390 #endif /* WITH_JMAP */
5391
5392 #ifdef WITH_DAV
mailbox_delete_caldav(struct mailbox * mailbox)5393 static int mailbox_delete_caldav(struct mailbox *mailbox)
5394 {
5395 struct caldav_db *caldavdb = NULL;
5396
5397 caldavdb = caldav_open_mailbox(mailbox);
5398 if (caldavdb) {
5399 int r = caldav_delmbox(caldavdb, mailbox->name);
5400 caldav_close(caldavdb);
5401 if (r) return r;
5402 }
5403
5404 int r = caldav_alarm_delete_mailbox(mailbox->name);
5405 if (r) return r;
5406
5407 return 0;
5408 }
5409
mailbox_delete_carddav(struct mailbox * mailbox)5410 static int mailbox_delete_carddav(struct mailbox *mailbox)
5411 {
5412 struct carddav_db *carddavdb = NULL;
5413
5414 carddavdb = carddav_open_mailbox(mailbox);
5415 if (carddavdb) {
5416 int r = carddav_delmbox(carddavdb, mailbox->name);
5417 carddav_close(carddavdb);
5418 if (r) return r;
5419 }
5420
5421 return 0;
5422 }
5423
mailbox_delete_webdav(struct mailbox * mailbox)5424 static int mailbox_delete_webdav(struct mailbox *mailbox)
5425 {
5426 struct webdav_db *webdavdb = NULL;
5427
5428 webdavdb = webdav_open_mailbox(mailbox);
5429 if (webdavdb) {
5430 int r = webdav_delmbox(webdavdb, mailbox->name);
5431 webdav_close(webdavdb);
5432 if (r) return r;
5433 }
5434
5435 return 0;
5436 }
5437
mailbox_delete_dav(struct mailbox * mailbox)5438 static int mailbox_delete_dav(struct mailbox *mailbox)
5439 {
5440 if (mailbox->mbtype & MBTYPE_ADDRESSBOOK)
5441 return mailbox_delete_carddav(mailbox);
5442 if (mailbox->mbtype & MBTYPE_CALENDAR)
5443 return mailbox_delete_caldav(mailbox);
5444 if (mailbox->mbtype & MBTYPE_COLLECTION)
5445 return mailbox_delete_webdav(mailbox);
5446 return 0;
5447 }
5448 #endif /* WITH_DAV */
5449
5450 /*
5451 * Delete and close the mailbox 'mailbox'. Closes 'mailbox' whether
5452 * or not the deletion was successful. Requires a locked mailbox.
5453 */
mailbox_delete(struct mailbox ** mailboxptr)5454 EXPORTED int mailbox_delete(struct mailbox **mailboxptr)
5455 {
5456 struct mailbox *mailbox = *mailboxptr;
5457 int r;
5458
5459 r = mailbox_delete_conversations(mailbox);
5460 if (r) return r;
5461
5462 #ifdef WITH_JMAP
5463 r = mailbox_delete_alarms(mailbox);
5464 if (r) return r;
5465 #endif /* WITH_JMAP */
5466
5467 #ifdef WITH_DAV
5468 r = mailbox_delete_dav(mailbox);
5469 if (r) return r;
5470 #endif /* WITH_DAV */
5471
5472 return mailbox_delete_internal(mailboxptr);
5473 }
5474
5475 struct meta_file {
5476 unsigned long metaflag;
5477 int optional;
5478 int nolink;
5479 };
5480
5481 static struct meta_file meta_files[] = {
5482 { META_HEADER, 0, 1 },
5483 { META_INDEX, 0, 1 },
5484 { META_CACHE, 1, 1 },
5485 { META_SQUAT, 1, 0 },
5486 { META_ANNOTATIONS, 1, 1 },
5487 { META_ARCHIVECACHE, 1, 1 },
5488 { 0, 0, 0 }
5489 };
5490
5491 /* XXX - move this part of cleanup into mboxlist. Really
5492 * needs to be done with mailboxes.db locked so nobody can
5493 * try to create a mailbox while the delete is underway.
5494 * VERY tight race condition exists right now... */
5495 /* we need an exclusive namelock for this */
mailbox_delete_cleanup(struct mailbox * mailbox,const char * part,const char * name,const char * uniqueid)5496 HIDDEN int mailbox_delete_cleanup(struct mailbox *mailbox, const char *part, const char *name, const char *uniqueid)
5497 {
5498 strarray_t paths = STRARRAY_INITIALIZER;
5499 int i;
5500 mbentry_t *mbentry;
5501 struct meta_file *mf;
5502 int r;
5503 char nbuf[MAX_MAILBOX_NAME];
5504 char *ntail;
5505 char *p;
5506
5507 int object_storage_enabled = 0 ;
5508 #if defined ENABLE_OBJECTSTORE
5509 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED);
5510 #endif
5511
5512 xstrncpy(nbuf, name, sizeof(nbuf));
5513 ntail = nbuf + strlen(nbuf);
5514
5515 if (mailbox && object_storage_enabled){
5516 mailbox_remove_files_from_object_storage (mailbox, 0) ;
5517 }
5518
5519 /* XXX - double XXX - this is a really ugly function. It should be
5520 * using mboxname_parts and walking back up the 'boxes' list */
5521
5522 strarray_add(&paths, mboxname_datapath(part, name, uniqueid, 0));
5523
5524 /* find the directory for every one of the meta files */
5525 for (mf = meta_files; mf->metaflag; mf++) {
5526 char *fname = xstrdup(mboxname_metapath(part, name, uniqueid, mf->metaflag, 0));
5527 p = strrchr(fname, '/');
5528 if (p) *p = '\0';
5529 strarray_add(&paths, fname);
5530 free(fname);
5531 }
5532
5533 for (i = 0; i < paths.count; i++) {
5534 const char *path = strarray_nth(&paths, i);
5535 mailbox_delete_files(path);
5536 }
5537
5538 do {
5539 /* Check if the mailbox has children */
5540 r = mboxlist_mboxtree(nbuf, chkchildren, (void *)part, MBOXTREE_SKIP_ROOT);
5541 if (r != 0) break; /* We short-circuit with CYRUSDB_DONE */
5542
5543 /* no children, remove the directories */
5544 for (i = 0; i < paths.count; i++) {
5545 char *path = paths.data[i]; /* need direct reference, because we're fiddling */
5546 r = rmdir(path);
5547 if (r && errno != ENOENT)
5548 syslog(LOG_NOTICE,
5549 "Remove of supposedly empty directory %s failed: %m",
5550 path);
5551 p = strrchr(path, '/');
5552 if (p) *p = '\0';
5553 }
5554
5555 /* Check if parent mailbox exists */
5556 ntail = strrchr(nbuf, '.');
5557 if (!ntail || strchr(ntail, '!')) {
5558 /* Hit top of hierarchy or domain separator */
5559 break;
5560 }
5561 *ntail = '\0';
5562 if (!strcmp(nbuf, "user") ||
5563 ((ntail - nbuf > 5) && !strcmp(ntail-5, "!user"))) {
5564 /* Hit top of 'user' hierarchy */
5565 break;
5566 }
5567
5568 r = mboxlist_lookup(nbuf, &mbentry, NULL);
5569 /* if it's not being moved, and not the same partition, then it's safe to
5570 * clean up the parent directory too */
5571 if (!r) {
5572 if (!(mbentry->mbtype & MBTYPE_MOVING) && strcmp(mbentry->partition, part))
5573 r = IMAP_MAILBOX_NONEXISTENT;
5574 mboxlist_entry_free(&mbentry);
5575 }
5576 } while (r == IMAP_MAILBOX_NONEXISTENT);
5577
5578 strarray_fini(&paths);
5579
5580 return 0;
5581 }
5582
mailbox_copy_files(struct mailbox * mailbox,const char * newpart,const char * newname,const char * newuniqueid)5583 EXPORTED int mailbox_copy_files(struct mailbox *mailbox, const char *newpart,
5584 const char *newname, const char *newuniqueid)
5585 {
5586 char oldbuf[MAX_MAILBOX_PATH], newbuf[MAX_MAILBOX_PATH];
5587 struct meta_file *mf;
5588 const message_t *msg;
5589 int r = 0;
5590
5591 int object_storage_enabled = 0 ;
5592 #if defined ENABLE_OBJECTSTORE
5593 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED);
5594 #endif
5595
5596 /* Copy over meta files */
5597 for (mf = meta_files; mf->metaflag; mf++) {
5598 struct stat sbuf;
5599
5600 xstrncpy(oldbuf, mailbox_meta_fname(mailbox, mf->metaflag),
5601 MAX_MAILBOX_PATH);
5602 xstrncpy(newbuf, mboxname_metapath(newpart, newname, newuniqueid, mf->metaflag, 0),
5603 MAX_MAILBOX_PATH);
5604
5605 unlink(newbuf); /* Make link() possible */
5606
5607 if (!mf->optional || stat(oldbuf, &sbuf) != -1) {
5608 r = mailbox_copyfile(oldbuf, newbuf, mf->nolink);
5609 if (r) return r;
5610 }
5611 }
5612
5613 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
5614 while ((msg = mailbox_iter_step(iter))) {
5615 const struct index_record *record = msg_record(msg);
5616 xstrncpy(oldbuf, mailbox_record_fname(mailbox, record),
5617 MAX_MAILBOX_PATH);
5618 if (!object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED)
5619 xstrncpy(newbuf, mboxname_archivepath(newpart, newname, newuniqueid, record->uid),
5620 MAX_MAILBOX_PATH);
5621 else
5622 xstrncpy(newbuf, mboxname_datapath(newpart, newname, newuniqueid, record->uid),
5623 MAX_MAILBOX_PATH);
5624
5625 if (!(object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED)) // if object storage do not move file
5626 r = mailbox_copyfile(oldbuf, newbuf, 0);
5627
5628 if (r) break;
5629
5630 #if defined ENABLE_OBJECTSTORE
5631 if (object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED) {
5632 static struct mailbox new_mailbox;
5633 memset(&new_mailbox, 0, sizeof(struct mailbox));
5634 new_mailbox.name = (char*) newname;
5635 new_mailbox.part = (char*) config_defpartition ;
5636 r = objectstore_put(&new_mailbox, record, newbuf); // put should just add to refcount.
5637 if (r) break;
5638 }
5639 #endif
5640 }
5641 mailbox_iter_done(&iter);
5642
5643 return r;
5644 }
5645
5646 /* if 'userid' is set, we perform the funky RENAME INBOX INBOX.old
5647 semantics, regardless of whether or not the name of the mailbox is
5648 'user.foo'.*/
5649 /* requires a write-locked oldmailbox pointer, since we delete it
5650 immediately afterwards */
mailbox_rename_copy(struct mailbox * oldmailbox,const char * newname,const char * newpartition,unsigned uidvalidity,const char * userid,int ignorequota,int silent,struct mailbox ** newmailboxptr)5651 HIDDEN int mailbox_rename_copy(struct mailbox *oldmailbox,
5652 const char *newname,
5653 const char *newpartition,
5654 unsigned uidvalidity,
5655 const char *userid, int ignorequota,
5656 int silent,
5657 struct mailbox **newmailboxptr)
5658 {
5659 int r;
5660 struct mailbox *newmailbox = NULL;
5661 struct conversations_state *oldcstate = NULL;
5662 struct conversations_state *newcstate = NULL;
5663 char *newquotaroot = NULL;
5664
5665 assert(mailbox_index_islocked(oldmailbox, 1));
5666
5667 /* we can't rename back from a deleted mailbox, because the conversations
5668 * information will be wrong. Ideally we might re-calculate, but for now
5669 * we just throw a big fat error */
5670 if (config_getswitch(IMAPOPT_CONVERSATIONS) &&
5671 mboxname_isdeletedmailbox(oldmailbox->name, NULL)) {
5672 syslog(LOG_ERR, "can't rename a deleted mailbox %s", oldmailbox->name);
5673 return IMAP_MAILBOX_BADNAME;
5674 }
5675
5676 /* create uidvalidity if not explicitly requested */
5677 if (!uidvalidity)
5678 uidvalidity = mboxname_nextuidvalidity(newname, oldmailbox->i.uidvalidity);
5679
5680 modseq_t highestmodseq = silent ? oldmailbox->i.highestmodseq : 0;
5681
5682 /* Create new mailbox */
5683 r = mailbox_create(newname, oldmailbox->mbtype, newpartition,
5684 oldmailbox->acl, (userid ? NULL : oldmailbox->uniqueid),
5685 oldmailbox->i.options, uidvalidity,
5686 oldmailbox->i.createdmodseq,
5687 highestmodseq, &newmailbox);
5688
5689 if (r) return r;
5690
5691 /* Check quota if necessary */
5692 if (!ignorequota && newmailbox->quotaroot &&
5693 strcmpsafe(oldmailbox->quotaroot, newmailbox->quotaroot)) {
5694
5695 quota_t usage[QUOTA_NUMRESOURCES];
5696 mailbox_get_usage(oldmailbox, usage);
5697 r = mailbox_quota_check(newmailbox, usage);
5698 /* then we abort - no space to rename */
5699 if (r)
5700 goto fail;
5701 }
5702 newquotaroot = xstrdupnull(newmailbox->quotaroot);
5703
5704 /* XXX - calculate new uniqueid somehow */
5705 r = mailbox_copy_files(oldmailbox, newpartition, newname, NULL);
5706 if (r) goto fail;
5707
5708 /* Re-open index file */
5709 r = mailbox_open_index(newmailbox);
5710 if (r) goto fail;
5711
5712 /* Re-lock index */
5713 r = mailbox_lock_index_internal(newmailbox, LOCK_EXCLUSIVE);
5714
5715 /* INBOX rename - change uniqueid */
5716 if (userid) {
5717 mailbox_make_uniqueid(newmailbox);
5718
5719 r = seen_copy(userid, oldmailbox, newmailbox);
5720 if (r) goto fail;
5721 }
5722
5723 /* copy any mailbox annotations (but keep the known quota
5724 * amount, because we already counted that usage. XXX horrible
5725 * hack */
5726 quota_t annotused = newmailbox->i.quota_annot_used;
5727 r = annotate_rename_mailbox(oldmailbox, newmailbox);
5728 if (r) goto fail;
5729 newmailbox->i.quota_annot_used = annotused;
5730
5731 /* mark the "used" back to zero, so it updates the new quota! */
5732 mailbox_set_quotaroot(newmailbox, newquotaroot);
5733 mailbox_quota_dirty(newmailbox);
5734 memset(newmailbox->quota_previously_used, 0, sizeof(newmailbox->quota_previously_used));
5735 /* except this one... because we've counted it when we created the folder */
5736 newmailbox->quota_previously_used[QUOTA_NUMFOLDERS] = 1;
5737
5738 /* re-set the UIDVALIDITY, it will have been the old one in the index header */
5739 mailbox_index_dirty(newmailbox);
5740 newmailbox->i.uidvalidity = uidvalidity;
5741 /* unless on a replica, bump the modseq too */
5742 if (!silent) mailbox_modseq_dirty(newmailbox);
5743
5744 /* NOTE: in the case of renaming a user to another user, we
5745 * don't rename the conversations DB - instead we re-create
5746 * the records in the target user. Sorry, was too complex
5747 * otherwise handling all the special cases */
5748 if (mailbox_has_conversations(oldmailbox)) {
5749 oldcstate = conversations_get_mbox(oldmailbox->name);
5750 assert(oldcstate);
5751 }
5752
5753 if (mailbox_has_conversations(newmailbox)) {
5754 newcstate = conversations_get_mbox(newmailbox->name);
5755 assert(newcstate);
5756 }
5757
5758 if (oldcstate && newcstate && !strcmp(oldcstate->path, newcstate->path)) {
5759 /* we can just rename within the same user */
5760 r = conversations_rename_folder(oldcstate, oldmailbox->name, newname);
5761 }
5762 else {
5763 /* have to handle each one separately */
5764 if (newcstate)
5765 r = mailbox_add_conversations(newmailbox, /*silent*/0);
5766 if (oldcstate)
5767 r = mailbox_delete_conversations(oldmailbox);
5768 }
5769 if (r) goto fail;
5770
5771 /* commit the index changes */
5772 r = mailbox_commit(newmailbox);
5773 if (r) goto fail;
5774
5775 if (config_auditlog)
5776 syslog(LOG_NOTICE, "auditlog: rename sessionid=<%s> "
5777 "oldmailbox=<%s> newmailbox=<%s> uniqueid=<%s>",
5778 session_id(),
5779 oldmailbox->name, newname, newmailbox->uniqueid);
5780
5781 if (newmailboxptr) *newmailboxptr = newmailbox;
5782 else mailbox_close(&newmailbox);
5783 free(newquotaroot);
5784
5785 return 0;
5786
5787 fail:
5788 /* first unlock so we don't need to write anything new down */
5789 mailbox_unlock_index(newmailbox, NULL);
5790 /* then remove all the files */
5791 mailbox_delete_cleanup(NULL, newmailbox->part, newmailbox->name, newmailbox->uniqueid);
5792 /* and finally, abort */
5793 mailbox_abort(newmailbox);
5794 mailbox_close(&newmailbox);
5795 free(newquotaroot);
5796
5797 return r;
5798 }
5799
mailbox_rename_cleanup(struct mailbox ** mailboxptr,int isinbox)5800 EXPORTED int mailbox_rename_cleanup(struct mailbox **mailboxptr, int isinbox)
5801 {
5802
5803 int r = 0;
5804 struct mailbox *oldmailbox = *mailboxptr;
5805 char *name = xstrdup(oldmailbox->name);
5806
5807 if (isinbox) {
5808 /* Expunge old mailbox */
5809 r = mailbox_expunge(oldmailbox, expungeall, (char *)0, NULL, 0);
5810 if (!r) r = mailbox_commit(oldmailbox);
5811 mailbox_close(mailboxptr);
5812 } else {
5813 r = mailbox_delete_internal(mailboxptr);
5814 }
5815
5816 if (r) {
5817 syslog(LOG_CRIT,
5818 "Rename Failure during mailbox_rename_cleanup (%s), " \
5819 "potential leaked space (%s)", name,
5820 error_message(r));
5821 }
5822 free(name);
5823
5824 return r;
5825 }
5826
5827 /*
5828 * Copy (or link) the file 'from' to the file 'to'
5829 */
mailbox_copyfile(const char * from,const char * to,int nolink)5830 EXPORTED int mailbox_copyfile(const char *from, const char *to, int nolink)
5831 {
5832 int flags = COPYFILE_MKDIR|COPYFILE_KEEPTIME;
5833 if (nolink) flags |= COPYFILE_NOLINK;
5834
5835 if (mailbox_wait_cb) mailbox_wait_cb(mailbox_wait_cb_rock);
5836
5837 if (cyrus_copyfile(from, to, flags))
5838 return IMAP_IOERROR;
5839
5840 return 0;
5841 }
5842
5843 /* ---------------------------------------------------------------------- */
5844 /* RECONSTRUCT SUPPORT */
5845 /* ---------------------------------------------------------------------- */
5846
5847 #define UIDGROW 300
5848
5849 struct found_uid {
5850 uint32_t uid;
5851 unsigned isarchive:1;
5852 };
5853
5854 struct found_uids {
5855 struct found_uid *found;
5856 unsigned nalloc;
5857 unsigned nused;
5858 unsigned pos;
5859 };
5860 #define FOUND_UIDS_INITIALIZER \
5861 { NULL, 0, 0, 0 }
5862
sort_found(const void * a,const void * b)5863 static int sort_found(const void *a, const void *b)
5864 {
5865 struct found_uid *fa = (struct found_uid *)a;
5866 struct found_uid *fb = (struct found_uid *)b;
5867 if (fa->uid != fb->uid)
5868 return fa->uid - fb->uid;
5869 return fa->isarchive - fb->isarchive;
5870 }
5871
add_found(struct found_uids * ff,uint32_t uid,int isarchive)5872 static void add_found(struct found_uids *ff, uint32_t uid, int isarchive)
5873 {
5874 /* make sure there's space */
5875 if (ff->nused >= ff->nalloc) {
5876 ff->nalloc += UIDGROW;
5877 ff->found = xrealloc(ff->found, ff->nalloc * sizeof(ff->found[0]));
5878 }
5879 ff->found[ff->nused].uid = uid;
5880 ff->found[ff->nused].isarchive = !!isarchive;
5881 ff->nused++;
5882 }
5883
free_found(struct found_uids * ff)5884 static void free_found(struct found_uids *ff)
5885 {
5886 free(ff->found);
5887 ff->found = NULL;
5888 ff->nalloc = 0;
5889 ff->nused = 0;
5890 ff->pos = 0;
5891 }
5892
parse_datafilename(const char * name,uint32_t * uidp)5893 static int parse_datafilename(const char *name, uint32_t *uidp)
5894 {
5895 const char *p = name;
5896
5897 /* must be at least one digit */
5898 if (!cyrus_isdigit(*p)) return IMAP_MAILBOX_BADNAME;
5899 do {
5900 p++;
5901 } while cyrus_isdigit(*p);
5902
5903 /* has to end with a dot */
5904 if (*p != '.') return IMAP_MAILBOX_BADNAME;
5905 if (p[1]) return IMAP_MAILBOX_BADNAME;
5906
5907 return parseuint32(name, &p, uidp);
5908 }
5909
find_files(struct mailbox * mailbox,struct found_uids * files,int flags)5910 static int find_files(struct mailbox *mailbox, struct found_uids *files,
5911 int flags)
5912 {
5913 strarray_t paths = STRARRAY_INITIALIZER;
5914 DIR *dirp;
5915 struct dirent *dirent;
5916 uint32_t uid;
5917 const char *p;
5918 char buf[MAX_MAILBOX_PATH];
5919 struct stat sbuf;
5920 int r;
5921 int i;
5922
5923 strarray_add(&paths, mailbox_datapath(mailbox, 0));
5924
5925 #if defined ENABLE_OBJECTSTORE
5926 if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED))
5927 {
5928 uint32_t i , count = 0 ;
5929 struct message *list = get_list_of_message (mailbox, &count) ;
5930 for (i = 0; i < count; i++) {
5931 add_found(files, list [i].message_uid, 1 /* isarchive */ );
5932 }
5933 discard_list( ) ;
5934 }
5935 else
5936 #endif
5937 strarray_add(&paths, mboxname_archivepath(mailbox->part, mailbox->name, mailbox->uniqueid, 0));
5938
5939 for (i = 0; i < paths.count; i++) {
5940 const char *dirpath = strarray_nth(&paths, i);
5941 int isarchive = strcmp(dirpath, mailbox_datapath(mailbox, 0));
5942
5943 dirp = opendir(dirpath);
5944 if (!dirp) continue;
5945
5946 /* data directory is fine */
5947 while ((dirent = readdir(dirp)) != NULL) {
5948 p = dirent->d_name;
5949 if (*p == '.') continue; /* dot files */
5950 if (!strncmp(p, "cyrus.", 6)) continue; /* cyrus.* files */
5951
5952 r = parse_datafilename(p, &uid);
5953
5954 if (r) {
5955 /* check if it's a directory */
5956 snprintf(buf, MAX_MAILBOX_PATH, "%s/%s", dirpath, dirent->d_name);
5957 if (stat(buf, &sbuf) == -1) continue; /* ignore ephemeral */
5958 if (!S_ISDIR(sbuf.st_mode)) {
5959 if (!(flags & RECONSTRUCT_IGNORE_ODDFILES)) {
5960 printf("%s odd file %s\n", mailbox->name, buf);
5961 syslog(LOG_ERR, "%s odd file %s", mailbox->name, buf);
5962 if (flags & RECONSTRUCT_REMOVE_ODDFILES)
5963 unlink(buf);
5964 else {
5965 printf("run reconstruct with -O to remove odd files\n");
5966 syslog(LOG_ERR, "run reconstruct with -O to "
5967 "remove odd files");
5968 }
5969 }
5970 }
5971 }
5972 else {
5973 /* it's one of ours :) */
5974 add_found(files, uid, isarchive);
5975 }
5976 }
5977
5978 closedir(dirp);
5979 }
5980
5981 /* make sure UIDs are sorted for comparison */
5982 qsort(files->found, files->nused, sizeof(files->found[0]), sort_found);
5983
5984 strarray_fini(&paths);
5985
5986 return 0;
5987 }
5988
cleanup_stale_expunged(struct mailbox * mailbox)5989 static void cleanup_stale_expunged(struct mailbox *mailbox)
5990 {
5991 const char *fname;
5992 int expunge_fd = -1;
5993 const char *expunge_base = NULL;
5994 size_t expunge_len = 0; /* mapped size */
5995 unsigned long expunge_num;
5996 unsigned long emapnum;
5997 uint32_t erecno;
5998 uint32_t eversion;
5999 bit32 eoffset, expungerecord_size;
6000 const char *bufp;
6001 struct stat sbuf;
6002 int r;
6003
6004 /* it's always read-writes */
6005 fname = mailbox_meta_fname(mailbox, META_EXPUNGE);
6006 expunge_fd = open(fname, O_RDWR, 0);
6007 if (expunge_fd == -1)
6008 goto done; /* yay, no crappy expunge file */
6009
6010 /* boo - gotta read and find out the UIDs */
6011 r = fstat(expunge_fd, &sbuf);
6012 if (r == -1)
6013 goto done;
6014
6015 if (sbuf.st_size < INDEX_HEADER_SIZE)
6016 goto done;
6017
6018 map_refresh(expunge_fd, 1, &expunge_base,
6019 &expunge_len, sbuf.st_size, "expunge",
6020 mailbox->name);
6021
6022 /* use the expunge file's header information just in case
6023 * versions are skewed for some reason */
6024 eoffset = ntohl(*((bit32 *)(expunge_base+OFFSET_START_OFFSET)));
6025 expungerecord_size = ntohl(*((bit32 *)(expunge_base+OFFSET_RECORD_SIZE)));
6026
6027 /* bogus data at the start of the expunge file? */
6028 if (!eoffset || !expungerecord_size)
6029 goto done;
6030
6031 expunge_num = ntohl(*((bit32 *)(expunge_base+OFFSET_NUM_RECORDS)));
6032 eversion = ntohl(*((bit32 *)(expunge_base+OFFSET_MINOR_VERSION)));
6033 emapnum = (sbuf.st_size - eoffset) / expungerecord_size;
6034 if (emapnum < expunge_num) {
6035 expunge_num = emapnum;
6036 }
6037
6038 /* add every UID to the files list */
6039 for (erecno = 1; erecno <= expunge_num; erecno++) {
6040 struct index_record record;
6041 bufp = expunge_base + eoffset + (erecno-1)*expungerecord_size;
6042 mailbox_buf_to_index_record(bufp, eversion, &record, 0);
6043 record.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
6044 mailbox_record_cleanup(mailbox, &record);
6045 }
6046
6047 fname = mailbox_meta_fname(mailbox, META_EXPUNGE);
6048 unlink(fname);
6049
6050 done:
6051 if (expunge_base) map_free(&expunge_base, &expunge_len);
6052 xclose(expunge_fd);
6053 }
6054
6055 /* this is kind of like mailbox_create, but we try to rescue
6056 * what we can from the filesystem! */
mailbox_reconstruct_create(const char * name,struct mailbox ** mbptr)6057 static int mailbox_reconstruct_create(const char *name, struct mailbox **mbptr)
6058 {
6059 struct mailbox *mailbox = NULL;
6060 int options = config_getint(IMAPOPT_MAILBOX_DEFAULT_OPTIONS)
6061 | OPT_POP3_NEW_UIDL;
6062 mbentry_t *mbentry = NULL;
6063 struct mailboxlist *listitem;
6064 int r;
6065
6066 /* make sure it's not already open. Very odd, since we already
6067 * discovered it's not openable! */
6068 listitem = find_listitem(name);
6069 if (listitem) return IMAP_MAILBOX_LOCKED;
6070
6071 listitem = create_listitem(name);
6072 mailbox = &listitem->m;
6073
6074 /* if we can't get an exclusive lock first try, there's something
6075 * racy going on! */
6076 r = mboxname_lock(name, &listitem->l, LOCK_NONBLOCKING);
6077 if (r) goto done;
6078
6079 /* Start by looking up current data in mailbox list */
6080 /* XXX - no mboxlist entry? Can we recover? */
6081 r = mboxlist_lookup(name, &mbentry, NULL);
6082 if (r) goto done;
6083
6084 mailbox->part = xstrdup(mbentry->partition);
6085 mailbox->acl = xstrdup(mbentry->acl);
6086 mailbox->mbtype = mbentry->mbtype;
6087
6088 syslog(LOG_NOTICE, "create new mailbox %s", name);
6089
6090 /* Attempt to open index */
6091 r = mailbox_open_index(mailbox);
6092 if (!r) r = mailbox_read_index_header(mailbox);
6093 if (r) {
6094 printf("%s: failed to read index header\n", mailbox->name);
6095 syslog(LOG_ERR, "failed to read index header for %s", mailbox->name);
6096 /* no cyrus.index file at all - well, we're in a pickle!
6097 * no point trying to rescue anything else... */
6098 mailbox_close(&mailbox);
6099 r = mailbox_create(name, mbentry->mbtype, mbentry->partition, mbentry->acl,
6100 mbentry->uniqueid, options, 0, 0, 0, mbptr);
6101 mboxlist_entry_free(&mbentry);
6102 return r;
6103 }
6104
6105 mboxlist_entry_free(&mbentry);
6106
6107 /* read header, if it is not there, we need to create it */
6108 r = mailbox_read_header(mailbox, NULL);
6109 if (r) {
6110 /* Header failed to read - recreate it */
6111 printf("%s: failed to read header file\n", mailbox->name);
6112 syslog(LOG_ERR, "failed to read header file for %s", mailbox->name);
6113
6114 mailbox_make_uniqueid(mailbox);
6115 r = mailbox_commit(mailbox);
6116 if (r) goto done;
6117 }
6118
6119 if (mailbox->header_file_crc != mailbox->i.header_file_crc) {
6120 mailbox->i.header_file_crc = mailbox->header_file_crc;
6121 printf("%s: header file CRC mismatch, correcting\n", mailbox->name);
6122 syslog(LOG_ERR, "%s: header file CRC mismatch, correcting", mailbox->name);
6123 mailbox_index_dirty(mailbox);
6124 r = mailbox_commit(mailbox);
6125 if (r) goto done;
6126 }
6127
6128 done:
6129 if (r) mailbox_close(&mailbox);
6130 else *mbptr = mailbox;
6131
6132 return r;
6133 }
6134
mailbox_reconstruct_uniqueid(struct mailbox * mailbox,int flags)6135 static int mailbox_reconstruct_uniqueid(struct mailbox *mailbox, int flags)
6136 {
6137 int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
6138 mbentry_t *mbentry = NULL;
6139
6140 int r = mboxlist_lookup(mailbox->name, &mbentry, 0);
6141 if (r) return r;
6142
6143 if (strcmpsafe(mbentry->uniqueid, mailbox->uniqueid)) {
6144 printf("%s: update uniqueid from header %s => %s\n", mailbox->name,
6145 mbentry->uniqueid, mailbox->uniqueid);
6146 if (make_changes) {
6147 if ((flags & RECONSTRUCT_PREFER_MBOXLIST) && mbentry->uniqueid) {
6148 mailbox_set_uniqueid(mailbox, mbentry->uniqueid);
6149 }
6150 else {
6151 free(mbentry->uniqueid);
6152 mbentry->uniqueid = xstrdup(mailbox->uniqueid);
6153 r = mboxlist_update(mbentry, 0);
6154 }
6155 }
6156 }
6157
6158 mboxlist_entry_free(&mbentry);
6159
6160 return r;
6161 }
6162
mailbox_reconstruct_acl(struct mailbox * mailbox,int flags)6163 static int mailbox_reconstruct_acl(struct mailbox *mailbox, int flags)
6164 {
6165 int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
6166 char *acl = NULL;
6167 int r;
6168
6169 r = mailbox_read_header(mailbox, &acl);
6170 if (r) return r;
6171
6172 if (strcmp(mailbox->acl, acl)) {
6173 printf("%s: update acl from header %s => %s\n", mailbox->name,
6174 mailbox->acl, acl);
6175 if (make_changes) {
6176 mbentry_t *mbentry = NULL;
6177 r = mboxlist_lookup(mailbox->name, &mbentry, NULL);
6178 if (!r) {
6179 if ((flags & RECONSTRUCT_PREFER_MBOXLIST) && mbentry->acl) {
6180 mailbox_set_acl(mailbox, mbentry->acl);
6181 }
6182 else {
6183 free(mbentry->acl);
6184 mbentry->acl = xstrdup(acl);
6185 r = mboxlist_update(mbentry, 0);
6186 }
6187 }
6188 mboxlist_entry_free(&mbentry);
6189 }
6190 }
6191
6192 free(acl);
6193
6194 return r;
6195 }
6196
records_match(const char * mboxname,struct index_record * old,struct index_record * new)6197 static int records_match(const char *mboxname,
6198 struct index_record *old,
6199 struct index_record *new)
6200 {
6201 int i;
6202 int match = 1;
6203 int userflags_dirty = 0;
6204
6205 if (old->internaldate != new->internaldate) {
6206 printf("%s uid %u mismatch: internaldate\n",
6207 mboxname, new->uid);
6208 match = 0;
6209 }
6210 if (old->sentdate != new->sentdate) {
6211 printf("%s uid %u mismatch: sentdate\n",
6212 mboxname, new->uid);
6213 match = 0;
6214 }
6215 if (old->size != new->size) {
6216 printf("%s uid %u mismatch: size\n",
6217 mboxname, new->uid);
6218 match = 0;
6219 }
6220 if (old->header_size != new->header_size) {
6221 printf("%s uid %u mismatch: header_size\n",
6222 mboxname, new->uid);
6223 match = 0;
6224 }
6225 if (old->gmtime != new->gmtime) {
6226 printf("%s uid %u mismatch: gmtime\n",
6227 mboxname, new->uid);
6228 match = 0;
6229 }
6230 if (old->savedate != new->savedate) {
6231 printf("%s uid %u mismatch: savedate\n",
6232 mboxname, new->uid);
6233 match = 0;
6234 }
6235 if (old->createdmodseq != new->createdmodseq) {
6236 printf("%s uid %u mismatch: createdmodseq\n",
6237 mboxname, new->uid);
6238 match = 0;
6239 }
6240 if (old->system_flags != new->system_flags) {
6241 printf("%s uid %u mismatch: systemflags\n",
6242 mboxname, new->uid);
6243 match = 0;
6244 }
6245 for (i = 0; i < MAX_USER_FLAGS/32; i++) {
6246 if (old->user_flags[i] != new->user_flags[i])
6247 userflags_dirty = 1;
6248 }
6249 if (userflags_dirty) {
6250 printf("%s uid %u mismatch: userflags\n",
6251 mboxname, new->uid);
6252 match = 0;
6253 }
6254 if (!message_guid_equal(&old->guid, &new->guid)) {
6255 printf("%s uid %u mismatch: guid\n",
6256 mboxname, new->uid);
6257 match = 0;
6258 }
6259
6260 if (!match) {
6261 syslog(LOG_ERR, "%s uid %u record mismatch, rewriting",
6262 mboxname, new->uid);
6263 }
6264
6265 /* cache issues - don't print, probably just a version
6266 * upgrade... */
6267 if (old->cache_version != new->cache_version) {
6268 match = 0;
6269 }
6270 if (old->cache_crc != new->cache_crc) {
6271 match = 0;
6272 }
6273 if (cache_len(old) != cache_len(new)) {
6274 match = 0;
6275 }
6276 /* only compare cache records if size matches */
6277 else if (memcmp(cache_base(old), cache_base(new), cache_len(new))) {
6278 match = 0;
6279 }
6280
6281 return match;
6282 }
6283
mailbox_reconstruct_compare_update(struct mailbox * mailbox,struct index_record * record,bit32 * valid_user_flags,int flags,int have_file,struct found_uids * discovered)6284 static int mailbox_reconstruct_compare_update(struct mailbox *mailbox,
6285 struct index_record *record,
6286 bit32 *valid_user_flags,
6287 int flags, int have_file,
6288 struct found_uids *discovered)
6289 {
6290 const char *fname = mailbox_record_fname(mailbox, record);
6291 int r = 0;
6292 int i;
6293 struct index_record copy;
6294 struct stat sbuf;
6295 int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
6296 int re_parse = flags & RECONSTRUCT_ALWAYS_PARSE;
6297 int do_stat = flags & RECONSTRUCT_DO_STAT;
6298 int re_pack = 0;
6299 int did_stat = 0;
6300
6301 int remove_temp_spool_file = 0 ;
6302 #if defined ENABLE_OBJECTSTORE
6303 int object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
6304
6305 if (object_storage_enabled && record->system_flags & FLAG_INTERNAL_ARCHIVED){ // put the file on spool temporarily
6306 remove_temp_spool_file = 1;
6307 r = objectstore_get(mailbox, record, fname);
6308 }
6309 #endif
6310
6311 /* does the file actually exist? */
6312 if (have_file && do_stat) {
6313 if (stat(fname, &sbuf) == -1 || (sbuf.st_size == 0)) {
6314 have_file = 0;
6315 }
6316 else if (record->size != (unsigned) sbuf.st_size) {
6317 re_parse = 1;
6318 }
6319 did_stat = 1;
6320 }
6321
6322 if (!have_file) {
6323 /* well, that's OK if it's supposed to be missing! */
6324 if (record->internal_flags & FLAG_INTERNAL_UNLINKED) {
6325 r = 0 ;
6326 goto out;
6327 }
6328
6329 printf("%s uid %u not found\n", mailbox->name, record->uid);
6330 syslog(LOG_ERR, "%s uid %u not found", mailbox->name, record->uid);
6331
6332 if (!make_changes) {
6333 r = 0 ;
6334 goto out;
6335 }
6336
6337 /* otherwise we have issues, mark it unlinked */
6338 unlink(fname);
6339 record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
6340 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
6341 r = mailbox_rewrite_index_record(mailbox, record);
6342 goto out;
6343 }
6344
6345 if (mailbox_cacherecord_internal(mailbox, record, MBCACHE_NOPARSE) || record->crec.len == 0) {
6346 re_parse = 1;
6347 re_pack = 1; /* cache record will have to be rewritten */
6348 }
6349
6350 /* copy once the cache record is read in... */
6351 copy = *record;
6352
6353 if (!record->internaldate) {
6354 re_parse = 1;
6355 }
6356
6357 /* re-calculate all the "derived" fields by parsing the file on disk */
6358 if (re_parse) {
6359 /* set NULL in case parse finds a new value */
6360 record->internaldate = 0;
6361
6362 r = message_parse(fname, record);
6363 if (r) goto out;
6364
6365 /* unchanged, keep the old value */
6366 if (!record->internaldate)
6367 record->internaldate = copy.internaldate;
6368
6369 /* it's not the same message! */
6370 if (!message_guid_equal(&record->guid, ©.guid)) {
6371 int do_unlink = 0;
6372
6373 printf("%s uid %u guid mismatch\n",
6374 mailbox->name, record->uid);
6375 syslog(LOG_ERR, "%s uid %u guid mismatch",
6376 mailbox->name, record->uid);
6377
6378 if (!make_changes) {
6379 r = 0 ;
6380 goto out;
6381 }
6382
6383 if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) {
6384 /* already expunged, just unlink it */
6385 printf("%s uid %u already expunged, unlinking\n",
6386 mailbox->name, record->uid);
6387 syslog(LOG_ERR, "%s uid %u already expunged, unlinking",
6388 mailbox->name, record->uid);
6389 do_unlink = 1;
6390 }
6391 else if (flags & RECONSTRUCT_GUID_REWRITE) {
6392 /* treat this file as discovered */
6393 add_found(discovered, record->uid,
6394 (record->internal_flags & FLAG_INTERNAL_ARCHIVED));
6395 printf("%s uid %u marking for uid upgrade\n",
6396 mailbox->name, record->uid);
6397 syslog(LOG_ERR, "%s uid %u marking for uid upgrade",
6398 mailbox->name, record->uid);
6399 do_unlink = 1;
6400 }
6401 else if (flags & RECONSTRUCT_GUID_UNLINK) {
6402 printf("%s uid %u unlinking as requested with -U\n",
6403 mailbox->name, record->uid);
6404 syslog(LOG_ERR, "%s uid %u unlinking as requested with -U",
6405 mailbox->name, record->uid);
6406 do_unlink = 1;
6407 }
6408
6409 if (do_unlink) {
6410 /* rewrite with the original so we don't break the
6411 * expectation that GUID never changes */
6412 copy.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
6413 mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK;
6414 r = mailbox_rewrite_index_record(mailbox, ©);
6415 goto out;
6416 }
6417
6418 /* otherwise we just report it and move on - hopefully the
6419 * correct file can be restored from backup or something */
6420 printf("run reconstruct with -R to fix or -U to remove\n");
6421 syslog(LOG_ERR, "run reconstruct with -R to fix or -U to remove");
6422 r = 0 ;
6423 goto out;
6424 }
6425 }
6426
6427 if (!record->size) {
6428 /* dang, guess it failed to parse */
6429
6430 printf("%s uid %u failed to parse\n", mailbox->name, record->uid);
6431 syslog(LOG_ERR, "%s uid %u failed to parse", mailbox->name, record->uid);
6432
6433 if (!make_changes) {
6434 r = 0 ;
6435 goto out;
6436 }
6437
6438 /* otherwise we have issues, mark it unlinked */
6439 unlink(fname);
6440 record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
6441 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
6442 r = mailbox_rewrite_index_record(mailbox, record);
6443 goto out;
6444 }
6445
6446 /* get internaldate from the file if not set */
6447 if (!record->internaldate) {
6448 if (did_stat || stat(fname, &sbuf) != -1)
6449 record->internaldate = sbuf.st_mtime;
6450 else
6451 record->internaldate = time(NULL);
6452 }
6453
6454 /* XXX - conditions under which modseq or uid or internaldate could be bogus? */
6455 if (record->modseq > mailbox->i.highestmodseq) {
6456 printf("%s uid %u future modseq " MODSEQ_FMT " found\n",
6457 mailbox->name, record->uid, record->modseq);
6458 syslog(LOG_ERR, "%s uid %u future modseq " MODSEQ_FMT " found",
6459 mailbox->name, record->uid, record->modseq);
6460 mailbox_index_dirty(mailbox);
6461 mailbox->i.highestmodseq = mboxname_setmodseq(mailbox->name,
6462 record->modseq,
6463 mailbox->mbtype, /*dofolder*/0);
6464 }
6465
6466 if (record->uid > mailbox->i.last_uid) {
6467 printf("%s future uid %u found\n",
6468 mailbox->name, record->uid);
6469 syslog(LOG_ERR, "%s future uid %u found",
6470 mailbox->name, record->uid);
6471 mailbox_index_dirty(mailbox);
6472 mailbox->i.last_uid = record->uid;
6473 }
6474
6475 /* remove any user_flags that are missing from the header */
6476 for (i = 0; i < MAX_USER_FLAGS/32; i++) {
6477 record->user_flags[i] &= valid_user_flags[i];
6478 }
6479
6480 /* after all this - if it still matches in every respect, we don't need
6481 * to rewrite the record - just return */
6482 if (records_match(mailbox->name, ©, record)) {
6483 r = 0 ;
6484 goto out;
6485 }
6486
6487 /* XXX - inform of changes */
6488 if (!make_changes) {
6489 r = 0 ;
6490 goto out;
6491 }
6492
6493 /* rewrite the cache record */
6494 if (re_pack || !record->cache_offset || record->cache_crc != copy.cache_crc) {
6495 int32_t oldcrc = copy.cache_crc;
6496 int32_t newcrc = record->cache_crc;
6497 size_t oldoff = record->cache_offset;
6498 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
6499 record->cache_offset = 0;
6500 r = mailbox_append_cache(mailbox, record);
6501 if (r) goto out ;
6502 printf("%s rewrote cache for %u (offset %llu to %llu, crc %u to %u/%u)\n",
6503 mailbox->name, record->uid,
6504 (long long unsigned)oldoff, (long long unsigned)record->cache_offset,
6505 oldcrc, newcrc, record->cache_crc);
6506 syslog(LOG_NOTICE, "%s rewrote cache for %u (offset %llu to %llu, crc %u to %u/%u)",
6507 mailbox->name, record->uid,
6508 (long long unsigned)oldoff, (long long unsigned)record->cache_offset,
6509 oldcrc, newcrc, record->cache_crc);
6510 }
6511
6512 r = mailbox_rewrite_index_record(mailbox, record);
6513
6514 out:
6515 if (remove_temp_spool_file)
6516 remove(fname);
6517 return r;
6518 }
6519
6520
mailbox_reconstruct_append(struct mailbox * mailbox,uint32_t uid,int isarchive,int flags)6521 static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid, int isarchive,
6522 int flags)
6523 {
6524 /* XXX - support archived */
6525 const char *fname;
6526 int r = 0;
6527 struct index_record record;
6528 struct stat sbuf;
6529 int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
6530
6531 memset(&record, 0, sizeof(struct index_record));
6532
6533 int remove_temp_spool_file = 0;
6534 int object_storage_enabled = 0;
6535 #if defined ENABLE_OBJECTSTORE
6536 object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED);
6537 #endif
6538
6539 if (isarchive && !object_storage_enabled)
6540 fname = mboxname_archivepath(mailbox->part, mailbox->name, mailbox->uniqueid, uid);
6541 else
6542 fname = mboxname_datapath(mailbox->part, mailbox->name, mailbox->uniqueid, uid);
6543
6544 #if defined ENABLE_OBJECTSTORE
6545 if (object_storage_enabled)
6546 {
6547 uint32_t i , count = 0;
6548 struct message *list = get_list_of_message (mailbox, &count);
6549 for (i = 0; i < count; i++) {
6550 if (uid == list[i].message_uid){
6551
6552 record.uid = uid;
6553 record.guid = list[i].message_guid;
6554 remove_temp_spool_file = 1;
6555 r = objectstore_get(mailbox, &record, fname);
6556 break;
6557 }
6558 }
6559 discard_list();
6560 }
6561 #endif
6562
6563 /* possible if '0.' file exists */
6564 if (!uid) {
6565 /* filthy hack - copy the path to '1.' and replace 1 with 0 */
6566 char *hack;
6567 fname = mboxname_datapath(mailbox->part, mailbox->name, mailbox->uniqueid, 1);
6568 hack = (char *)fname;
6569 hack[strlen(fname)-2] = '0';
6570 }
6571
6572 if (stat(fname, &sbuf) == -1) r = IMAP_MAILBOX_NONEXISTENT;
6573 else if (sbuf.st_size == 0) r = IMAP_MAILBOX_NONEXISTENT;
6574
6575 /* no file, nothing to do! */
6576 if (r) {
6577 syslog(LOG_ERR, "%s uid %u not found", mailbox->name, uid);
6578 printf("%s uid %u not found", mailbox->name, uid);
6579 r = 0;
6580
6581 if (make_changes)
6582 unlink(fname);
6583
6584 goto out;
6585 }
6586
6587 r = message_parse(fname, &record);
6588 if (r) goto out;
6589
6590 if (isarchive)
6591 record.internal_flags |= FLAG_INTERNAL_ARCHIVED;
6592
6593 /* copy the timestamp from the file if not calculated */
6594 if (!record.internaldate)
6595 record.internaldate = sbuf.st_mtime;
6596
6597 if (uid > mailbox->i.last_uid) {
6598 printf("%s uid %u found - adding\n", mailbox->name, uid);
6599 syslog(LOG_ERR, "%s uid %u found - adding", mailbox->name, uid);
6600 record.uid = uid;
6601 }
6602 else {
6603 char *oldfname;
6604 char *newfname;
6605
6606 printf("%s uid %u rediscovered - appending\n", mailbox->name, uid);
6607 syslog(LOG_ERR, "%s uid %u rediscovered - appending", mailbox->name, uid);
6608 /* XXX - check firstexpunged? */
6609 record.uid = mailbox->i.last_uid + 1;
6610
6611 if (!make_changes) {
6612 r = 0 ;
6613 goto out;
6614 }
6615
6616 oldfname = xstrdup(fname);
6617 newfname = xstrdup(mailbox_record_fname(mailbox, &record));
6618 r = rename(oldfname, newfname);
6619 free(oldfname);
6620 free(newfname);
6621 if (r) {
6622 r = IMAP_IOERROR;
6623 goto out ;
6624 }
6625 }
6626
6627 /* XXX - inform of changes */
6628 if (!make_changes){
6629 r = 0 ;
6630 goto out;
6631 }
6632
6633 r = mailbox_append_index_record(mailbox, &record);
6634
6635 /* XXX - copy per-message annotations? */
6636
6637 out:
6638 if (remove_temp_spool_file)
6639 remove(fname);
6640 return r;
6641 }
6642
6643
reconstruct_compare_headers(struct mailbox * mailbox,struct index_header * old,struct index_header * new)6644 static void reconstruct_compare_headers(struct mailbox *mailbox,
6645 struct index_header *old,
6646 struct index_header *new)
6647 {
6648 if (old->quota_mailbox_used != new->quota_mailbox_used) {
6649 printf("%s updating quota_mailbox_used: "
6650 QUOTA_T_FMT " => " QUOTA_T_FMT "\n", mailbox->name,
6651 old->quota_mailbox_used, new->quota_mailbox_used);
6652 syslog(LOG_ERR, "%s updating quota_mailbox_used: "
6653 QUOTA_T_FMT " => " QUOTA_T_FMT, mailbox->name,
6654 old->quota_mailbox_used, new->quota_mailbox_used);
6655 }
6656
6657 if (old->quota_annot_used != new->quota_annot_used) {
6658 printf("%s updating quota_annot_used: "
6659 QUOTA_T_FMT " => " QUOTA_T_FMT "\n", mailbox->name,
6660 old->quota_annot_used, new->quota_annot_used);
6661 syslog(LOG_ERR, "%s updating quota_annot_used: "
6662 QUOTA_T_FMT " => " QUOTA_T_FMT, mailbox->name,
6663 old->quota_annot_used, new->quota_annot_used);
6664 }
6665
6666 if (old->answered != new->answered) {
6667 syslog(LOG_ERR, "%s: updating answered %u => %u",
6668 mailbox->name, old->answered, new->answered);
6669 printf("%s: updating answered %u => %u\n",
6670 mailbox->name, old->answered, new->answered);
6671 }
6672
6673 if (old->flagged != new->flagged) {
6674 syslog(LOG_ERR, "%s: updating flagged %u => %u",
6675 mailbox->name, old->flagged, new->flagged);
6676 printf("%s: updating flagged %u => %u\n",
6677 mailbox->name, old->flagged, new->flagged);
6678 }
6679
6680 if (old->deleted != new->deleted) {
6681 syslog(LOG_ERR, "%s: updating deleted %u => %u",
6682 mailbox->name, old->deleted, new->deleted);
6683 printf("%s: updating deleted %u => %u\n",
6684 mailbox->name, old->deleted, new->deleted);
6685 }
6686
6687 if (old->exists != new->exists) {
6688 syslog(LOG_ERR, "%s: updating exists %u => %u",
6689 mailbox->name, old->exists, new->exists);
6690 printf("%s: updating exists %u => %u\n",
6691 mailbox->name, old->exists, new->exists);
6692 }
6693
6694 if (old->synccrcs.basic != new->synccrcs.basic) {
6695 syslog(LOG_ERR, "%s: updating sync_crc %u => %u",
6696 mailbox->name, old->synccrcs.basic, new->synccrcs.basic);
6697 printf("%s: updating sync_crc %u => %u\n",
6698 mailbox->name, old->synccrcs.basic, new->synccrcs.basic);
6699 }
6700
6701 if (old->synccrcs.annot != new->synccrcs.annot) {
6702 syslog(LOG_ERR, "%s: updating sync_crc_annot %u => %u",
6703 mailbox->name, old->synccrcs.annot, new->synccrcs.annot);
6704 printf("%s: updating sync_crc_annot %u => %u\n",
6705 mailbox->name, old->synccrcs.annot, new->synccrcs.annot);
6706 }
6707
6708 }
6709
mailbox_wipe_index_record(struct mailbox * mailbox,struct index_record * record)6710 static int mailbox_wipe_index_record(struct mailbox *mailbox,
6711 struct index_record *record)
6712 {
6713 int n;
6714 indexbuffer_t ibuf;
6715 unsigned char *buf = ibuf.buf;
6716 size_t offset;
6717
6718 assert(mailbox_index_islocked(mailbox, 1));
6719 assert(record->recno > 0 &&
6720 record->recno <= mailbox->i.num_records);
6721
6722 record->uid = 0;
6723 record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
6724
6725 mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
6726 mailbox_index_dirty(mailbox);
6727
6728 mailbox_index_record_to_buf(record, mailbox->i.minor_version, buf);
6729
6730 offset = mailbox->i.start_offset +
6731 (record->recno-1) * mailbox->i.record_size;
6732
6733 off_t p = lseek(mailbox->index_fd, offset, SEEK_SET);
6734 if (p == -1) {
6735 syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
6736 record->recno, mailbox->name);
6737 return IMAP_IOERROR;
6738 }
6739
6740 n = retry_write(mailbox->index_fd, buf, mailbox->i.record_size);
6741 if (n < 0) {
6742 syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
6743 record->recno, mailbox->name);
6744 return IMAP_IOERROR;
6745 }
6746
6747 return 0;
6748 }
6749
addannot_uid(const char * mailbox,uint32_t uid,const char * entry,const char * userid,const struct buf * value,const struct annotate_metadata * mdata,void * rock)6750 static int addannot_uid(const char *mailbox __attribute__((unused)),
6751 uint32_t uid,
6752 const char *entry __attribute__((unused)),
6753 const char *userid __attribute__((unused)),
6754 const struct buf *value __attribute__((unused)),
6755 const struct annotate_metadata *mdata __attribute__((unused)),
6756 void *rock)
6757 {
6758 struct found_uids *annots = (struct found_uids *)rock;
6759
6760 /* take advantage of the guarantee that all annotations with the same UID
6761 * will be together in a 'foreach' response */
6762 if (!annots->nused || annots->found[annots->nused-1].uid != uid) {
6763 /* we don't support an archive annotations DB yet */
6764 add_found(annots, uid, /*isarchive*/0);
6765 }
6766
6767 return 0;
6768 }
6769
6770
find_annots(struct mailbox * mailbox,struct found_uids * annots)6771 static int find_annots(struct mailbox *mailbox, struct found_uids *annots)
6772 {
6773 int r = 0;
6774
6775 r = annotatemore_findall(mailbox->name, ANNOTATE_ANY_UID, "*",
6776 /*modseq*/0, addannot_uid, annots, /*flags*/0);
6777 if (r) return r;
6778
6779 /* make sure UIDs are sorted for comparison */
6780 qsort(annots->found, annots->nused, sizeof(annots->found[0]), sort_found);
6781
6782 return 0;
6783 }
6784
reconstruct_delannots(struct mailbox * mailbox,struct found_uids * delannots,int flags)6785 static int reconstruct_delannots(struct mailbox *mailbox,
6786 struct found_uids *delannots,
6787 int flags)
6788 {
6789 int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES);
6790 int r = 0;
6791
6792 r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, NULL);
6793 if (r) {
6794 syslog(LOG_ERR, "IOERROR: failed to open annotations %s: %s",
6795 mailbox->name, error_message(r));
6796 goto out;
6797 }
6798
6799 while (delannots->pos < delannots->nused) {
6800 uint32_t uid = delannots->found[delannots->pos].uid;
6801 syslog(LOG_NOTICE, "removing stale annotations for %u", uid);
6802 printf("removing stale annotations for %u\n", uid);
6803 if (make_changes) {
6804 r = annotate_msg_cleanup(mailbox, uid);
6805 if (r) goto out;
6806 }
6807 delannots->pos++;
6808 }
6809
6810 out:
6811 return r;
6812 }
6813
6814
6815 /*
6816 * Reconstruct the single mailbox named 'name'
6817 */
mailbox_reconstruct(const char * name,int flags)6818 EXPORTED int mailbox_reconstruct(const char *name, int flags)
6819 {
6820 /* settings */
6821 int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES);
6822
6823 int r = 0;
6824 int i, flag;
6825 struct mailbox_iter *iter = NULL;
6826 struct mailbox *mailbox = NULL;
6827 struct found_uids files = FOUND_UIDS_INITIALIZER;
6828 struct found_uids discovered = FOUND_UIDS_INITIALIZER;
6829 struct found_uids annots = FOUND_UIDS_INITIALIZER;
6830 struct found_uids delannots = FOUND_UIDS_INITIALIZER;
6831 struct index_header old_header;
6832 int have_file;
6833 uint32_t last_seen_uid = 0;
6834 bit32 valid_user_flags[MAX_USER_FLAGS/32];
6835 struct buf buf = BUF_INITIALIZER;
6836
6837 if (make_changes && !(flags & RECONSTRUCT_QUIET)) {
6838 syslog(LOG_NOTICE, "reconstructing %s", name);
6839 }
6840
6841 r = mailbox_open_iwl(name, &mailbox);
6842 if (r) {
6843 if (!make_changes) return r;
6844 /* returns a locktype == LOCK_EXCLUSIVE mailbox */
6845 r = mailbox_reconstruct_create(name, &mailbox);
6846 }
6847 if (r) return r;
6848
6849 /* NOTE: we have to do this first, because it reads the header */
6850 r = mailbox_reconstruct_acl(mailbox, flags);
6851 if (r) goto close;
6852
6853 r = mailbox_reconstruct_uniqueid(mailbox, flags);
6854 if (r) goto close;
6855
6856 /* open and lock the annotation state */
6857 r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, NULL);
6858 if (r) {
6859 syslog(LOG_ERR, "IOERROR: failed to open annotations %s: %s",
6860 mailbox->name, error_message(r));
6861 goto close;
6862 }
6863
6864 /* Validate user flags */
6865 for (i = 0; i < MAX_USER_FLAGS/32; i++) {
6866 valid_user_flags[i] = 0;
6867 }
6868 for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
6869 if (!mailbox->flagname[flag]) continue;
6870 if ((flag && !mailbox->flagname[flag-1]) ||
6871 !imparse_isatom(mailbox->flagname[flag])) {
6872 printf("%s: bogus flag name %d:%s",
6873 mailbox->name, flag, mailbox->flagname[flag]);
6874 syslog(LOG_ERR, "%s: bogus flag name %d:%s",
6875 mailbox->name, flag, mailbox->flagname[flag]);
6876 mailbox->header_dirty = 1;
6877 free(mailbox->flagname[flag]);
6878 mailbox->flagname[flag] = NULL;
6879 continue;
6880 }
6881 valid_user_flags[flag/32] |= 1<<(flag&31);
6882 }
6883
6884 /* find cyrus.expunge file if present */
6885 cleanup_stale_expunged(mailbox);
6886
6887 r = find_files(mailbox, &files, flags);
6888 if (r) goto close;
6889
6890 r = find_annots(mailbox, &annots);
6891 if (r) goto close;
6892
6893 uint32_t recno;
6894 struct index_record record;
6895 for (recno = 1; recno <= mailbox->i.num_records; recno++) {
6896 r = mailbox_read_index_record(mailbox, recno, &record);
6897 if (r || record.uid <= last_seen_uid) {
6898 if (!r && record.uid)
6899 syslog(LOG_ERR, "%s out of order uid %u at record %u, wiping",
6900 mailbox->name, record.uid, record.recno);
6901 if (r)
6902 syslog(LOG_ERR, "%s failed to read at record %u (%s), wiping",
6903 mailbox->name, record.recno, error_message(r));
6904 mailbox_wipe_index_record(mailbox, &record);
6905 continue;
6906 }
6907
6908 last_seen_uid = record.uid;
6909
6910 /* bogus annotations? */
6911 while (annots.pos < annots.nused && annots.found[annots.pos].uid < record.uid) {
6912 add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0);
6913 annots.pos++;
6914 }
6915
6916 /* skip over current */
6917 while (annots.pos < annots.nused && annots.found[annots.pos].uid == record.uid) {
6918 annots.pos++;
6919 }
6920
6921 /* lower UID file exists */
6922 while (files.pos < files.nused && files.found[files.pos].uid < record.uid) {
6923 add_found(&discovered, files.found[files.pos].uid, files.found[files.pos].isarchive);
6924 files.pos++;
6925 }
6926
6927 /* if they match, advance the pointer */
6928 have_file = 0;
6929 while (files.pos < files.nused && files.found[files.pos].uid == record.uid) {
6930 if (have_file) {
6931 /* we can just unlink this one, already processed one copy */
6932 const char *fname = mboxname_archivepath(mailbox->part, mailbox->name,
6933 mailbox->uniqueid, record.uid);
6934 printf("Removing duplicate archive file %s\n", fname);
6935 unlink(fname);
6936 }
6937 else {
6938 if (files.found[files.pos].isarchive) {
6939 if (!(record.internal_flags & FLAG_INTERNAL_ARCHIVED)) {
6940 /* oops, it's really archived - let's fix that right now */
6941 record.internal_flags |= FLAG_INTERNAL_ARCHIVED;
6942 printf("Marking file as archived %s %u\n", mailbox->name, record.uid);
6943 mailbox_rewrite_index_record(mailbox, &record);
6944 }
6945 }
6946 else {
6947 if (record.internal_flags & FLAG_INTERNAL_ARCHIVED) {
6948 /* oops, non-archived copy exists, let's use that */
6949 record.internal_flags &= ~FLAG_INTERNAL_ARCHIVED;
6950 printf("Marking file as not archived %s %u\n", mailbox->name, record.uid);
6951 mailbox_rewrite_index_record(mailbox, &record);
6952 }
6953 }
6954 have_file = 1;
6955 }
6956 files.pos++;
6957 }
6958
6959 r = mailbox_reconstruct_compare_update(mailbox, &record,
6960 valid_user_flags,
6961 flags, have_file,
6962 &discovered);
6963 if (r) goto close;
6964
6965 if (mailbox->i.minor_version >= 13) {
6966 buf_reset(&buf);
6967 mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "thrid", "", &buf);
6968 if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "thrid", NULL, &buf);
6969 if (buf.len) {
6970 syslog(LOG_NOTICE, "removing stale thrid for %u", record.uid);
6971 printf("removing stale thrid for %u\n", record.uid);
6972 buf_reset(&buf);
6973 r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "thrid", "", &buf);
6974 if (r) goto close;
6975 }
6976 }
6977
6978 if (mailbox->i.minor_version >= 15) {
6979 buf_reset(&buf);
6980 mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "savedate", "", &buf);
6981 if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "savedate", NULL, &buf);
6982 if (buf.len) {
6983 syslog(LOG_NOTICE, "removing stale savedate for %u", record.uid);
6984 printf("removing stale savedate for %u\n", record.uid);
6985 buf_reset(&buf);
6986 r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "savedate", "", &buf);
6987 if (r) goto close;
6988 }
6989 }
6990
6991 if (mailbox->i.minor_version >= 16) {
6992 buf_reset(&buf);
6993 mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", "", &buf);
6994 if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", NULL, &buf);
6995 if (buf.len) {
6996 syslog(LOG_NOTICE, "removing stale createdmodseq for %u", record.uid);
6997 printf("removing stale createdmodseq for %u\n", record.uid);
6998 buf_reset(&buf);
6999 r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", "", &buf);
7000 if (r) goto close;
7001 }
7002 }
7003 }
7004
7005 /* add discovered messages before last_uid to the list in order */
7006 while (files.pos < files.nused && files.found[files.pos].uid <= mailbox->i.last_uid) {
7007 add_found(&discovered, files.found[files.pos].uid, files.found[files.pos].isarchive);
7008 files.pos++;
7009 }
7010
7011 /* messages AFTER last_uid can keep the same UID (see also, restore
7012 * from lost .index file) - so don't bother moving those */
7013 while (files.pos < files.nused) {
7014 uint32_t uid = files.found[files.pos].uid;
7015 r = mailbox_reconstruct_append(mailbox, files.found[files.pos].uid,
7016 files.found[files.pos].isarchive, flags);
7017 if (r) goto close;
7018 files.pos++;
7019
7020 /* we can keep this annotation too... */
7021
7022 /* bogus annotations? */
7023 while (annots.pos < annots.nused && annots.found[annots.pos].uid < uid) {
7024 add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0);
7025 annots.pos++;
7026 }
7027
7028 /* skip over current */
7029 while (annots.pos < annots.nused && annots.found[annots.pos].uid == uid) {
7030 annots.pos++;
7031 }
7032 }
7033
7034 /* bogus annotations after the end? */
7035 while (annots.pos < annots.nused) {
7036 add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0);
7037 annots.pos++;
7038 }
7039
7040 /* handle new list - note, we don't copy annotations for these */
7041 while (discovered.pos < discovered.nused) {
7042 r = mailbox_reconstruct_append(mailbox, discovered.found[discovered.pos].uid,
7043 discovered.found[discovered.pos].isarchive, flags);
7044 if (r) goto close;
7045 discovered.pos++;
7046 }
7047
7048 if (delannots.nused) {
7049 r = reconstruct_delannots(mailbox, &delannots, flags);
7050 if (r) goto close;
7051 }
7052
7053 /* make sure we have enough index file mmaped */
7054 r = mailbox_refresh_index_map(mailbox);
7055
7056 old_header = mailbox->i;
7057
7058 /* re-calculate derived fields */
7059 r = mailbox_index_recalc(mailbox);
7060 if (r) goto close;
7061
7062 /* inform users of any changed header fields */
7063 reconstruct_compare_headers(mailbox, &old_header, &mailbox->i);
7064
7065 /* fix up 2.4.0 bug breakage */
7066 if (!mailbox->i.uidvalidity) {
7067 if (make_changes) {
7068 mailbox->i.uidvalidity = mboxname_nextuidvalidity(mailbox->name, time(0));
7069 mailbox_index_dirty(mailbox);
7070 }
7071 syslog(LOG_ERR, "%s: zero uidvalidity", mailbox->name);
7072 }
7073 if (!mailbox->i.highestmodseq) {
7074 if (make_changes) {
7075 mailbox_index_dirty(mailbox);
7076 mailbox->i.highestmodseq = mboxname_nextmodseq(mailbox->name, 0, mailbox->mbtype, /*dofolder*/1);
7077 }
7078 syslog(LOG_ERR, "%s: zero highestmodseq", mailbox->name);
7079 }
7080
7081 if (make_changes) {
7082 r = mailbox_commit(mailbox);
7083 }
7084 else {
7085 r = mailbox_abort(mailbox);
7086 }
7087
7088 close:
7089 mailbox_iter_done(&iter);
7090 free_found(&files);
7091 free_found(&discovered);
7092 free_found(&annots);
7093 free_found(&delannots);
7094 mailbox_close(&mailbox);
7095 buf_free(&buf);
7096 return r;
7097 }
7098
mailbox_iter_init(struct mailbox * mailbox,modseq_t changedsince,unsigned flags)7099 EXPORTED struct mailbox_iter *mailbox_iter_init(struct mailbox *mailbox,
7100 modseq_t changedsince,
7101 unsigned flags)
7102 {
7103 struct mailbox_iter *iter = xzmalloc(sizeof(struct mailbox_iter));
7104 iter->mailbox = mailbox;
7105 iter->changedsince = changedsince;
7106 iter->num_records = mailbox->i.num_records;
7107 iter->msg = message_new();
7108
7109 /* calculate which system_flags to skip over */
7110 if (flags & ITER_SKIP_UNLINKED)
7111 iter->skipflags |= FLAG_INTERNAL_UNLINKED;
7112 if (flags & ITER_SKIP_EXPUNGED)
7113 iter->skipflags |= FLAG_INTERNAL_EXPUNGED;
7114 if (flags & ITER_SKIP_DELETED)
7115 iter->skipflags |= FLAG_DELETED;
7116
7117 return iter;
7118 }
7119
mailbox_iter_startuid(struct mailbox_iter * iter,uint32_t uid)7120 EXPORTED void mailbox_iter_startuid(struct mailbox_iter *iter, uint32_t uid)
7121 {
7122 struct mailbox *mailbox = iter->mailbox;
7123 iter->recno = uid ? mailbox_finduid(mailbox, uid-1) : 0;
7124 }
7125
mailbox_iter_step(struct mailbox_iter * iter)7126 EXPORTED const message_t *mailbox_iter_step(struct mailbox_iter *iter)
7127 {
7128 if (mailbox_wait_cb) mailbox_wait_cb(mailbox_wait_cb_rock);
7129
7130 for (iter->recno++; iter->recno <= iter->num_records; iter->recno++) {
7131 message_set_from_mailbox(iter->mailbox, iter->recno, iter->msg);
7132 const struct index_record *record = msg_record(iter->msg);
7133 if (!record->uid) continue; /* can happen on damaged mailboxes */
7134 if ((record->system_flags & iter->skipflags)) continue;
7135 if ((record->internal_flags & iter->skipflags)) continue;
7136 if (iter->changedsince && record->modseq <= iter->changedsince) continue;
7137 return iter->msg;
7138 }
7139
7140 /* guess we're done */
7141 return NULL;
7142 }
7143
mailbox_iter_done(struct mailbox_iter ** iterp)7144 EXPORTED void mailbox_iter_done(struct mailbox_iter **iterp)
7145 {
7146 struct mailbox_iter *iter = *iterp;
7147 if (!iter) return;
7148 message_unref(&iter->msg);
7149 free(iter);
7150 *iterp = NULL;
7151 }
7152
7153 /*
7154 * Gets messages usage.
7155 */
mailbox_get_usage(struct mailbox * mailbox,quota_t usage[QUOTA_NUMRESOURCES])7156 EXPORTED void mailbox_get_usage(struct mailbox *mailbox,
7157 quota_t usage[QUOTA_NUMRESOURCES])
7158 {
7159 int res;
7160
7161 for (res = 0; res < QUOTA_NUMRESOURCES; res++) {
7162 usage[res] = 0;
7163 }
7164
7165 if (!(mailbox->i.options & OPT_MAILBOX_DELETED)) {
7166 usage[QUOTA_STORAGE] = mailbox->i.quota_mailbox_used;
7167 usage[QUOTA_MESSAGE] = mailbox->i.exists;
7168 usage[QUOTA_ANNOTSTORAGE] = mailbox->i.quota_annot_used;
7169 usage[QUOTA_NUMFOLDERS] = 1;
7170 }
7171 /* else: mailbox is being deleted, thus its new usage is 0 */
7172 }
7173
mailbox_get_annotate_state(struct mailbox * mailbox,unsigned int uid,annotate_state_t ** statep)7174 EXPORTED int mailbox_get_annotate_state(struct mailbox *mailbox,
7175 unsigned int uid,
7176 annotate_state_t **statep)
7177 {
7178 int r = 0;
7179
7180 if (statep) *statep = NULL;
7181
7182 if (!mailbox->annot_state)
7183 mailbox->annot_state = annotate_state_new();
7184
7185 r = annotate_state_set_message(mailbox->annot_state, mailbox, uid);
7186 if (r) return r;
7187
7188 /* lock immediately if we have a write lock */
7189 if (mailbox_index_islocked(mailbox, /*write*/1))
7190 annotate_state_begin(mailbox->annot_state);
7191
7192 if (statep) *statep = mailbox->annot_state;
7193
7194 return 0;
7195 }
7196
mailbox_annotation_write(struct mailbox * mailbox,uint32_t uid,const char * entry,const char * userid,const struct buf * value)7197 EXPORTED int mailbox_annotation_write(struct mailbox *mailbox, uint32_t uid,
7198 const char *entry, const char *userid,
7199 const struct buf *value)
7200 {
7201 annotate_state_t *state = NULL;
7202 int r = 0;
7203 struct buf oldvalue = BUF_INITIALIZER;
7204
7205 annotatemore_msg_lookup(mailbox->name, uid, entry, userid, &oldvalue);
7206 if (oldvalue.len == value->len && (!value->len || !memcmp(oldvalue.s, value->s, value->len)))
7207 goto done;
7208
7209 struct index_record record;
7210 memset(&record, 0, sizeof(struct index_record));
7211 r = mailbox_find_index_record(mailbox, uid, &record);
7212 if (r) goto done;
7213
7214 r = mailbox_get_annotate_state(mailbox, uid, &state);
7215 if (r) goto done;
7216
7217 r = annotate_state_write(state, entry, userid, value);
7218 if (r) goto done;
7219
7220 /* need to touch the modseq */
7221 r = mailbox_rewrite_index_record(mailbox, &record);
7222 if (r) goto done;
7223
7224 done:
7225 buf_free(&oldvalue);
7226 return r;
7227 }
7228
mailbox_annotation_writemask(struct mailbox * mailbox,uint32_t uid,const char * entry,const char * userid,const struct buf * value)7229 EXPORTED int mailbox_annotation_writemask(struct mailbox *mailbox, uint32_t uid,
7230 const char *entry, const char *userid,
7231 const struct buf *value)
7232 {
7233 annotate_state_t *state = NULL;
7234 int r = 0;
7235 struct buf oldvalue = BUF_INITIALIZER;
7236
7237 /* we don't lookupmask here - because we want to still write the value as the
7238 * user's own value rather than the masked value, regardless of whether they
7239 * have the same content */
7240 annotatemore_msg_lookup(mailbox->name, uid, entry, userid, &oldvalue);
7241 if (oldvalue.len == value->len && (!value->len || !memcmp(oldvalue.s, value->s, value->len)))
7242 goto done;
7243
7244 struct index_record record;
7245 memset(&record, 0, sizeof(struct index_record));
7246 r = mailbox_find_index_record(mailbox, uid, &record);
7247 if (r) goto done;
7248
7249 r = mailbox_get_annotate_state(mailbox, uid, &state);
7250 if (r) goto done;
7251
7252 r = annotate_state_writemask(state, entry, userid, value);
7253 if (r) goto done;
7254
7255 /* need to touch the modseq */
7256 r = mailbox_rewrite_index_record(mailbox, &record);
7257 if (r) goto done;
7258
7259 done:
7260 buf_free(&oldvalue);
7261 return r;
7262 }
7263
mailbox_annotation_lookup(struct mailbox * mailbox,uint32_t uid,const char * entry,const char * userid,struct buf * value)7264 EXPORTED int mailbox_annotation_lookup(struct mailbox *mailbox, uint32_t uid,
7265 const char *entry, const char *userid,
7266 struct buf *value)
7267 {
7268 return annotatemore_msg_lookup(mailbox->name, uid, entry, userid, value);
7269 }
7270
mailbox_annotation_lookupmask(struct mailbox * mailbox,uint32_t uid,const char * entry,const char * userid,struct buf * value)7271 EXPORTED int mailbox_annotation_lookupmask(struct mailbox *mailbox, uint32_t uid,
7272 const char *entry, const char *userid,
7273 struct buf *value)
7274 {
7275 return annotatemore_msg_lookupmask(mailbox->name, uid, entry, userid, value);
7276 }
7277
7278
mailbox_cid_rename(struct mailbox * mailbox,conversation_id_t from_cid,conversation_id_t to_cid)7279 int mailbox_cid_rename(struct mailbox *mailbox,
7280 conversation_id_t from_cid,
7281 conversation_id_t to_cid)
7282 {
7283 const message_t *msg;
7284 int r = 0;
7285
7286 if (!config_getswitch(IMAPOPT_CONVERSATIONS))
7287 return 0;
7288
7289 struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
7290 while ((msg = mailbox_iter_step(iter))) {
7291 const struct index_record *record = msg_record(msg);
7292 if (record->cid != from_cid)
7293 continue;
7294
7295 /*
7296 * Just rename the CID in place - injecting a copy at the end
7297 * messes with clients that just use UID ordering, like Apple's
7298 * IOS email client */
7299
7300 struct index_record copyrecord = *record;
7301 copyrecord.cid = to_cid;
7302 r = mailbox_rewrite_index_record(mailbox, ©record);
7303
7304 if (r) {
7305 syslog(LOG_ERR, "mailbox_cid_rename: error "
7306 "rewriting record %u, mailbox %s: %s from %llu to %llu",
7307 record->recno, mailbox->name, error_message(r), from_cid, to_cid);
7308 break;
7309 }
7310 }
7311 mailbox_iter_done(&iter);
7312
7313 return r;
7314 }
7315
mailbox_set_wait_cb(void (* cb)(void *),void * rock)7316 EXPORTED void mailbox_set_wait_cb(void (*cb)(void *), void *rock)
7317 {
7318 mailbox_wait_cb = cb;
7319 mailbox_wait_cb_rock = rock;
7320 }
7321
7322 /* if either CRC is zero for a field, then we consider it to match.
7323 * this lets us bootstrap the case where CRCs weren't being calculated,
7324 * and also allows a client with incomplete local information to request
7325 * a change be made on a sync_server without having to fetch all the
7326 * data first just to calculate the CRC */
mailbox_crceq(struct synccrcs a,struct synccrcs b)7327 EXPORTED int mailbox_crceq(struct synccrcs a, struct synccrcs b)
7328 {
7329 if (a.basic && b.basic && a.basic != b.basic) return 0;
7330 if (a.annot && b.annot && a.annot != b.annot) return 0;
7331 return 1;
7332 }
7333