/* statuscache_db.c -- Status caching routines * * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include "assert.h" #include "cyrusdb.h" #include "imapd.h" #include "global.h" #include "mboxlist.h" #include "mailbox.h" #include "seen.h" #include "util.h" #include "xmalloc.h" #include "xstrlcpy.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" #include "statuscache.h" #define DB config_statuscache_db static struct db *statuscachedb = NULL; static int _initted = 0; /********************* CACHE METHODS ***********************/ static void statuscache_open(void) { if (!config_getswitch(IMAPOPT_STATUSCACHE)) return; char *fname = xstrdupnull(config_getstring(IMAPOPT_STATUSCACHE_DB_PATH)); if (!fname) fname = strconcat(config_dir, FNAME_STATUSCACHEDB, (char *)NULL); int r = cyrusdb_open(DB, fname, CYRUSDB_CREATE, &statuscachedb); if (r) { syslog(LOG_ERR, "DBERROR: opening %s: %s", fname, cyrusdb_strerror(r)); syslog(LOG_ERR, "statuscache in degraded mode"); statuscachedb = NULL; } free(fname); } static void statuscache_close(void) { if (!statuscachedb) return; int r = cyrusdb_close(statuscachedb); if (r) { syslog(LOG_ERR, "DBERROR: error closing statuscache: %s", cyrusdb_strerror(r)); } statuscachedb = NULL; } static void done_cb(void *rock __attribute__((unused))) { statuscache_close(); } static void init_internal() { if (_initted) return; statuscache_open(); cyrus_modules_add(done_cb, NULL); _initted = 1; } static void statuscache_buildkey(const char *mboxname, const char *userid, struct buf *buf) { buf_setcstr(buf, mboxname); /* double % is a safe separator, it can't exist in a mailboxname */ buf_putc(buf, '%'); if (userid) { buf_putc(buf, '%'); buf_appendcstr(buf, userid); } } static void statuscache_read_index(const char *mboxname, struct statusdata *sdata) { struct buf keybuf = BUF_INITIALIZER; const char *data = NULL; size_t datalen = 0; /* Don't access DB if it hasn't been opened */ if (!statuscachedb) return; /* Check if there is an entry in the database */ statuscache_buildkey(mboxname, NULL, &keybuf); int r = cyrusdb_fetch(statuscachedb, keybuf.s, keybuf.len, &data, &datalen, NULL); buf_free(&keybuf); if (r || !data || !datalen) return; const char *dend = data + datalen; char *p = (char *)data; if (*p++ != 'I') return; if (*p++ != ' ') return; unsigned version = (unsigned) strtoul(p, &p, 10); if (version != (unsigned) STATUSCACHE_VERSION) { /* Wrong version */ return; } if (*p++ != ' ') return; if (*p++ != '(') return; // read the matched items if (p < dend) sdata->messages = strtoul(p, &p, 10); if (p < dend) sdata->uidnext = strtoul(p, &p, 10); if (p < dend) sdata->uidvalidity = strtoul(p, &p, 10); if (p < dend) sdata->mboptions = strtoul(p, &p, 10); if (p < dend) sdata->size = strtoull(p, &p, 10); if (p < dend) sdata->createdmodseq = strtoull(p, &p, 10); if (p < dend) sdata->highestmodseq = strtoull(p, &p, 10); if (*p++ != ')') return; /* Sanity check the data */ if (!sdata->highestmodseq) return; sdata->statusitems |= STATUS_INDEXITEMS | STATUS_UIDVALIDITY; } static void statuscache_read_seen(const char *mboxname, const char *userid, struct statusdata *sdata) { struct buf keybuf = BUF_INITIALIZER; const char *data = NULL; size_t datalen = 0; if (!userid) return; // if no messages, other counts must also be zero if (!sdata->messages) { sdata->recent = 0; sdata->unseen = 0; sdata->userid = userid; sdata->statusitems |= STATUS_SEENITEMS; return; } /* Don't access DB if it hasn't been opened */ if (!statuscachedb) return; // we must have a HIGHESTMODSEQ to compare against if (!(sdata->statusitems & STATUS_HIGHESTMODSEQ)) return; /* Check if there is an entry in the database */ statuscache_buildkey(mboxname, userid, &keybuf); int r = cyrusdb_fetch(statuscachedb, keybuf.s, keybuf.len, &data, &datalen, NULL); buf_free(&keybuf); if (r || !data || !datalen) return; const char *dend = data + datalen; char *p = (char *)data; if (*p++ != 'S') return; if (*p++ != ' ') return; unsigned version = (unsigned) strtoul(p, &p, 10); if (version != (unsigned) STATUSCACHE_VERSION) { /* Wrong version */ return; } if (*p++ != ' ') return; if (*p++ != '(') return; // read the matched items if (p < dend) sdata->recent = strtoul(p, &p, 10); if (p < dend) sdata->unseen = strtoul(p, &p, 10); modseq_t highestmodseq = strtoull(p, &p, 10); if (*p++ != ')') return; // doesn't match non-unseen key if (highestmodseq != sdata->highestmodseq) return; sdata->userid = userid; sdata->statusitems |= STATUS_SEENITEMS; } static int statuscache_lookup(const char *mboxname, const char *userid, unsigned statusitems, struct statusdata *sdata) { // nothing to read! if (!(statusitems & (STATUS_INDEXITEMS|STATUS_SEENITEMS))) return 0; init_internal(); statuscache_read_index(mboxname, sdata); if (statusitems & STATUS_SEENITEMS) statuscache_read_seen(mboxname, userid, sdata); // did we get everything we wanted? if ((sdata->statusitems & statusitems) != statusitems) return IMAP_NO_NOSUCHMSG; return 0; } static int statuscache_store(const char *mboxname, struct statusdata *sdata, struct txn **tidptr) { struct buf keybuf = BUF_INITIALIZER; struct buf databuf = BUF_INITIALIZER; int r = 0; statuscache_buildkey(mboxname, /*userid*/NULL, &keybuf); /* if we don't have a full index, just nuke the key */ if (!sdata || (sdata->statusitems & STATUS_INDEXITEMS) != STATUS_INDEXITEMS) { r = cyrusdb_delete(statuscachedb, keybuf.s, keybuf.len, tidptr, 1); if (r != CYRUSDB_OK) { syslog(LOG_ERR, "DBERROR: error deleting statuscache for: %s (%s)", mboxname, cyrusdb_strerror(r)); } goto done; } buf_printf(&databuf, "I %u (%u %u %u %u %llu " MODSEQ_FMT " " MODSEQ_FMT ")", STATUSCACHE_VERSION, sdata->messages, sdata->uidnext, sdata->uidvalidity, sdata->mboptions, sdata->size, sdata->createdmodseq, sdata->highestmodseq); r = cyrusdb_store(statuscachedb, keybuf.s, keybuf.len, databuf.s, databuf.len, tidptr); if (r != CYRUSDB_OK) { syslog(LOG_ERR, "DBERROR: error updating database: %s (%s)", mboxname, cyrusdb_strerror(r)); goto done; } if ((sdata->statusitems & STATUS_SEENITEMS) != STATUS_SEENITEMS) goto done; // if there's no userid, we don't store this stuff if (!sdata->userid) goto done; statuscache_buildkey(mboxname, sdata->userid, &keybuf); /* The trailing whitespace is necessary because we * use non-length-based functions to parse the values. * Any non-digit char would be fine, but whitespace * looks less ugly in dbtool output */ buf_reset(&databuf); buf_printf(&databuf, "S %u (%u %u " MODSEQ_FMT ")", STATUSCACHE_VERSION, sdata->recent, sdata->unseen, sdata->highestmodseq); r = cyrusdb_store(statuscachedb, keybuf.s, keybuf.len, databuf.s, databuf.len, tidptr); if (r != CYRUSDB_OK) { syslog(LOG_ERR, "DBERROR: error updating database: %s (%s)", mboxname, cyrusdb_strerror(r)); goto done; } done: buf_free(&keybuf); buf_free(&databuf); return r; } HIDDEN int statuscache_invalidate(const char *mboxname, struct statusdata *sdata) { int doclose = 0; struct txn *tid = NULL; /* if it's disabled then skip */ if (!config_getswitch(IMAPOPT_STATUSCACHE)) return 0; /* if it's not already open, open and close it for just this */ if (!statuscachedb) { statuscache_open(); // failed to open, oh well if (!statuscachedb) return 0; doclose = 1; } int r = statuscache_store(mboxname, sdata, &tid); if (!r) { cyrusdb_commit(statuscachedb, tid); } else { syslog(LOG_NOTICE, "DBERROR: failed to store statuscache data for %s", mboxname); if (tid) cyrusdb_abort(statuscachedb, tid); } // if we opened the DB, close it now if (doclose) statuscache_close(); return 0; } /****************** STATUSDATA FILLING METHODS ************************/ HIDDEN void status_fill_mbentry(const mbentry_t *mbentry, struct statusdata *sdata) { assert(mbentry); assert(sdata); sdata->uidvalidity = mbentry->uidvalidity; sdata->mailboxid = mbentry->uniqueid; sdata->statusitems |= STATUS_MBENTRYITEMS; } HIDDEN void status_fill_mailbox(struct mailbox *mailbox, struct statusdata *sdata) { assert(mailbox); assert(sdata); sdata->messages = mailbox->i.exists; sdata->uidnext = mailbox->i.last_uid+1; sdata->mboptions = mailbox->i.options; sdata->size = mailbox->i.quota_mailbox_used; sdata->createdmodseq = mailbox->i.createdmodseq; sdata->highestmodseq = mailbox->i.highestmodseq; // mbentry items are also available from an open mailbox sdata->uidvalidity = mailbox->i.uidvalidity; sdata->mailboxid = mailbox->uniqueid; sdata->statusitems |= STATUS_INDEXITEMS | STATUS_MBENTRYITEMS; } HIDDEN void status_fill_seen(const char *userid, struct statusdata *sdata, unsigned numrecent, unsigned numunseen) { assert(userid); assert(sdata); // we need a matching parent record to exist for these values to be valid assert(sdata->statusitems & STATUS_HIGHESTMODSEQ); sdata->userid = userid; sdata->recent = numrecent; sdata->unseen = numunseen; sdata->statusitems |= STATUS_SEENITEMS; } static int status_load_mailbox(struct mailbox *mailbox, const char *userid, unsigned statusitems, struct statusdata *sdata) { status_fill_mailbox(mailbox, sdata); if ((statusitems & STATUS_SEENITEMS) && mailbox->i.exists) { unsigned numrecent = 0; unsigned numunseen = 0; /* Read \Seen state */ struct seqset *seq = NULL; int internalseen = mailbox_internal_seen(mailbox, userid); unsigned recentuid; if (internalseen) { recentuid = mailbox->i.recentuid; } else { struct seen *seendb = NULL; struct seendata sd = SEENDATA_INITIALIZER; int r = seen_open(userid, SEEN_CREATE, &seendb); if (!r) r = seen_read(seendb, mailbox->uniqueid, &sd); seen_close(&seendb); if (r) return r; recentuid = sd.lastuid; seq = seqset_parse(sd.seenuids, NULL, recentuid); seen_freedata(&sd); } struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); const message_t *msg; while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (record->uid > recentuid) numrecent++; if (internalseen) { if (!(record->system_flags & FLAG_SEEN)) numunseen++; } else { if (!seqset_ismember(seq, record->uid)) numunseen++; } } mailbox_iter_done(&iter); seqset_free(seq); status_fill_seen(userid, sdata, numrecent, numunseen); } statuscache_invalidate(mailbox->name, sdata); return 0; } static int status_lookup_internal(const char *mboxname, const char *userid, unsigned statusitems, struct statusdata *sdata) { struct mailbox *mailbox = NULL; int r = 0; /* Check status cache if possible */ if (config_getswitch(IMAPOPT_STATUSCACHE)) { /* Do actual lookup of cache item. */ r = statuscache_lookup(mboxname, userid, statusitems, sdata); /* Seen/recent status uses "push" invalidation events from * seen_db.c. This avoids needing to open cyrus.header to get * the mailbox uniqueid to open the seen db and get the * unseen_mtime and recentuid. */ if (!r) { syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'", mboxname, userid, statusitems); return 0; } syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'", mboxname, userid, statusitems); } /* Missing or invalid cache entry */ r = mailbox_open_irl(mboxname, &mailbox); if (r) return r; r = status_load_mailbox(mailbox, userid, statusitems, sdata); /* cache the new value while unlocking */ if (!r) mailbox_unlock_index(mailbox, sdata); mailbox_close(&mailbox); return r; } EXPORTED int status_lookup_mbentry(const mbentry_t *mbentry, const char *userid, unsigned statusitems, struct statusdata *sdata) { // check if we can get everything we need from the mbentry status_fill_mbentry(mbentry, sdata); if ((sdata->statusitems & statusitems) == statusitems) return 0; return status_lookup_internal(mbentry->name, userid, statusitems, sdata); } EXPORTED int status_lookup_mboxname(const char *mboxname, const char *userid, unsigned statusitems, struct statusdata *sdata) { // we want an mbentry first, just in case we can get everything from there if (statusitems & STATUS_MAILBOXID) { mbentry_t *mbentry = NULL; int r = mboxlist_lookup_allow_all(mboxname, &mbentry, NULL); if (r) return r; r = status_lookup_mbentry(mbentry, userid, statusitems, sdata); mboxlist_entry_free(&mbentry); return r; } return status_lookup_internal(mboxname, userid, statusitems, sdata); } // this one has literally no smarts at all EXPORTED int status_lookup_mbname(const mbname_t *mbname, const char *userid, unsigned statusitems, struct statusdata *sdata) { return status_lookup_mboxname(mbname_intname(mbname), userid, statusitems, sdata); } /* * Performs a STATUS command on an open mailbox */ EXPORTED int status_lookup_mailbox(struct mailbox *mailbox, const char *userid, unsigned statusitems, struct statusdata *sdata) { // check if we already have all the data we need (includes any possible mbentry) status_fill_mailbox(mailbox, sdata); if ((sdata->statusitems & statusitems) == statusitems) return 0; /* Check status cache if possible */ if (config_getswitch(IMAPOPT_STATUSCACHE)) { /* Do actual lookup of cache item. */ int r = statuscache_lookup(mailbox->name, userid, statusitems, sdata); /* Seen/recent status uses "push" invalidation events from * seen_db.c. This avoids needing to open cyrus.header to get * the mailbox uniqueid to open the seen db and get the * unseen_mtime and recentuid. */ if (!r) { syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'", mailbox->name, userid, statusitems); return 0; } syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'", mailbox->name, userid, statusitems); } return status_load_mailbox(mailbox, userid, statusitems, sdata); }