cyrus-imapd-3.4.2/imap/statuscache_db.c

/* statuscache_db.c -- Status caching routines
 *
 * Copyright (c) 1994-2008 Carnegie Mellon University.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The name "Carnegie Mellon University" must not be used to
 *    endorse or promote products derived from this software without
 *    prior written permission. For permission or any legal
 *    details, please contact
 *      Carnegie Mellon University
 *      Center for Technology Transfer and Enterprise Creation
 *      4615 Forbes Avenue
 *      Suite 302
 *      Pittsburgh, PA  15213
 *      (412) 268-7393, fax: (412) 268-7395
 *      innovation@andrew.cmu.edu
 *
 * 4. Redistributions of any form whatsoever must retain the following
 *    acknowledgment:
 *    "This product includes software developed by Computing Services
 *     at Carnegie Mellon University (http://www.cmu.edu/computing/)."
 *
 * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
 * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
 * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <fcntl.h>
#include <syslog.h>

#include "assert.h"
#include "cyrusdb.h"
#include "imapd.h"
#include "global.h"
#include "mboxlist.h"
#include "mailbox.h"
#include "seen.h"
#include "util.h"
#include "xmalloc.h"
#include "xstrlcpy.h"

/* generated headers are not necessarily in current directory */
#include "imap/imap_err.h"

#include "statuscache.h"

#define DB config_statuscache_db

static struct db *statuscachedb = NULL;
static int _initted = 0;

/********************* CACHE METHODS ***********************/

static void statuscache_open(void)
{
    if (!config_getswitch(IMAPOPT_STATUSCACHE))
        return;

    char *fname = xstrdupnull(config_getstring(IMAPOPT_STATUSCACHE_DB_PATH));
    if (!fname)
        fname = strconcat(config_dir, FNAME_STATUSCACHEDB, (char *)NULL);

    int r = cyrusdb_open(DB, fname, CYRUSDB_CREATE, &statuscachedb);
    if (r) {
        syslog(LOG_ERR, "DBERROR: opening %s: %s", fname,
               cyrusdb_strerror(r));
        syslog(LOG_ERR, "statuscache in degraded mode");
        statuscachedb = NULL;
    }

    free(fname);
}

static void statuscache_close(void)
{
    if (!statuscachedb) return;

    int r = cyrusdb_close(statuscachedb);
    if (r) {
        syslog(LOG_ERR, "DBERROR: error closing statuscache: %s",
              cyrusdb_strerror(r));
    }

    statuscachedb = NULL;
}

static void done_cb(void *rock __attribute__((unused)))
{
    statuscache_close();
}

static void init_internal()
{
    if (_initted) return;
    statuscache_open();
    cyrus_modules_add(done_cb, NULL);
    _initted = 1;
}

static void statuscache_buildkey(const char *mboxname, const char *userid,
                                 struct buf *buf)
{
    buf_setcstr(buf, mboxname);
    /* double % is a safe separator, it can't exist in a mailboxname */
    buf_putc(buf, '%');
    if (userid) {
        buf_putc(buf, '%');
        buf_appendcstr(buf, userid);
    }
}

static void statuscache_read_index(const char *mboxname, struct statusdata *sdata)
{
    struct buf keybuf = BUF_INITIALIZER;
    const char *data = NULL;
    size_t datalen = 0;

    /* Don't access DB if it hasn't been opened */
    if (!statuscachedb)
        return;

    /* Check if there is an entry in the database */
    statuscache_buildkey(mboxname, NULL, &keybuf);
    int r = cyrusdb_fetch(statuscachedb, keybuf.s, keybuf.len, &data, &datalen, NULL);
    buf_free(&keybuf);

    if (r || !data || !datalen)
        return;

    const char *dend = data + datalen;

    char *p = (char *)data;
    if (*p++ != 'I') return;
    if (*p++ != ' ') return;

    unsigned version = (unsigned) strtoul(p, &p, 10);
    if (version != (unsigned) STATUSCACHE_VERSION) {
        /* Wrong version */
        return;
    }

    if (*p++ != ' ') return;
    if (*p++ != '(') return;

    // read the matched items
    if (p < dend) sdata->messages = strtoul(p, &p, 10);
    if (p < dend) sdata->uidnext = strtoul(p, &p, 10);
    if (p < dend) sdata->uidvalidity = strtoul(p, &p, 10);
    if (p < dend) sdata->mboptions = strtoul(p, &p, 10);
    if (p < dend) sdata->size = strtoull(p, &p, 10);
    if (p < dend) sdata->createdmodseq = strtoull(p, &p, 10);
    if (p < dend) sdata->highestmodseq = strtoull(p, &p, 10);

    if (*p++ != ')') return;

    /* Sanity check the data */
    if (!sdata->highestmodseq)
        return;

    sdata->statusitems |= STATUS_INDEXITEMS | STATUS_UIDVALIDITY;
}

static void statuscache_read_seen(const char *mboxname, const char *userid,
                                  struct statusdata *sdata)
{
    struct buf keybuf = BUF_INITIALIZER;
    const char *data = NULL;
    size_t datalen = 0;

    if (!userid)
        return;

    // if no messages, other counts must also be zero
    if (!sdata->messages) {
        sdata->recent = 0;
        sdata->unseen = 0;
        sdata->userid = userid;
        sdata->statusitems |= STATUS_SEENITEMS;
        return;
    }

    /* Don't access DB if it hasn't been opened */
    if (!statuscachedb)
        return;

    // we must have a HIGHESTMODSEQ to compare against
    if (!(sdata->statusitems & STATUS_HIGHESTMODSEQ))
        return;

    /* Check if there is an entry in the database */
    statuscache_buildkey(mboxname, userid, &keybuf);
    int r = cyrusdb_fetch(statuscachedb, keybuf.s, keybuf.len, &data, &datalen, NULL);
    buf_free(&keybuf);

    if (r || !data || !datalen)
        return;

    const char *dend = data + datalen;
    char *p = (char *)data;
    if (*p++ != 'S') return;
    if (*p++ != ' ') return;

    unsigned version = (unsigned) strtoul(p, &p, 10);
    if (version != (unsigned) STATUSCACHE_VERSION) {
        /* Wrong version */
        return;
    }

    if (*p++ != ' ') return;
    if (*p++ != '(') return;

    // read the matched items
    if (p < dend) sdata->recent = strtoul(p, &p, 10);
    if (p < dend) sdata->unseen = strtoul(p, &p, 10);
    modseq_t highestmodseq = strtoull(p, &p, 10);

    if (*p++ != ')') return;

    // doesn't match non-unseen key
    if (highestmodseq != sdata->highestmodseq)
        return;

    sdata->userid = userid;
    sdata->statusitems |= STATUS_SEENITEMS;
}

static int statuscache_lookup(const char *mboxname, const char *userid,
                       unsigned statusitems, struct statusdata *sdata)
{
    // nothing to read!
    if (!(statusitems & (STATUS_INDEXITEMS|STATUS_SEENITEMS)))
        return 0;

    init_internal();

    statuscache_read_index(mboxname, sdata);
    if (statusitems & STATUS_SEENITEMS)
        statuscache_read_seen(mboxname, userid, sdata);

    // did we get everything we wanted?
    if ((sdata->statusitems & statusitems) != statusitems)
        return IMAP_NO_NOSUCHMSG;

    return 0;
}

static int statuscache_store(const char *mboxname,
                             struct statusdata *sdata,
                             struct txn **tidptr)
{
    struct buf keybuf = BUF_INITIALIZER;
    struct buf databuf = BUF_INITIALIZER;
    int r = 0;

    statuscache_buildkey(mboxname, /*userid*/NULL, &keybuf);

    /* if we don't have a full index, just nuke the key */
    if (!sdata || (sdata->statusitems & STATUS_INDEXITEMS) != STATUS_INDEXITEMS) {
        r = cyrusdb_delete(statuscachedb, keybuf.s, keybuf.len, tidptr, 1);
        if (r != CYRUSDB_OK) {
            syslog(LOG_ERR, "DBERROR: error deleting statuscache for: %s (%s)",
                   mboxname, cyrusdb_strerror(r));
        }
        goto done;
    }


    buf_printf(&databuf,
                       "I %u (%u %u %u %u %llu " MODSEQ_FMT " " MODSEQ_FMT ")",
                       STATUSCACHE_VERSION,
                       sdata->messages, sdata->uidnext,
                       sdata->uidvalidity, sdata->mboptions, sdata->size,
                       sdata->createdmodseq, sdata->highestmodseq);

    r = cyrusdb_store(statuscachedb, keybuf.s, keybuf.len, databuf.s, databuf.len, tidptr);

    if (r != CYRUSDB_OK) {
        syslog(LOG_ERR, "DBERROR: error updating database: %s (%s)",
               mboxname, cyrusdb_strerror(r));
        goto done;
    }

    if ((sdata->statusitems & STATUS_SEENITEMS) != STATUS_SEENITEMS)
        goto done;

    // if there's no userid, we don't store this stuff
    if (!sdata->userid)
        goto done;

    statuscache_buildkey(mboxname, sdata->userid, &keybuf);

    /* The trailing whitespace is necessary because we
     * use non-length-based functions to parse the values.
     * Any non-digit char would be fine, but whitespace
     * looks less ugly in dbtool output */
    buf_reset(&databuf);
    buf_printf(&databuf,
                       "S %u (%u %u " MODSEQ_FMT ")",
                       STATUSCACHE_VERSION,
                       sdata->recent, sdata->unseen,
                       sdata->highestmodseq);

    r = cyrusdb_store(statuscachedb, keybuf.s, keybuf.len, databuf.s, databuf.len, tidptr);

    if (r != CYRUSDB_OK) {
        syslog(LOG_ERR, "DBERROR: error updating database: %s (%s)",
               mboxname, cyrusdb_strerror(r));
        goto done;
    }

done:
    buf_free(&keybuf);
    buf_free(&databuf);
    return r;
}

HIDDEN int statuscache_invalidate(const char *mboxname, struct statusdata *sdata)
{
    int doclose = 0;
    struct txn *tid = NULL;

    /* if it's disabled then skip */
    if (!config_getswitch(IMAPOPT_STATUSCACHE))
        return 0;

    /* if it's not already open, open and close it for just this */
    if (!statuscachedb) {
        statuscache_open();
        // failed to open, oh well
        if (!statuscachedb)
            return 0;
        doclose = 1;
    }

    int r = statuscache_store(mboxname, sdata, &tid);

    if (!r) {
        cyrusdb_commit(statuscachedb, tid);
    }
    else {
        syslog(LOG_NOTICE, "DBERROR: failed to store statuscache data for %s", mboxname);
        if (tid) cyrusdb_abort(statuscachedb, tid);
    }

    // if we opened the DB, close it now
    if (doclose)
        statuscache_close();

    return 0;
}


/****************** STATUSDATA FILLING METHODS ************************/

HIDDEN void status_fill_mbentry(const mbentry_t *mbentry, struct statusdata *sdata)
{
    assert(mbentry);
    assert(sdata);

    sdata->uidvalidity = mbentry->uidvalidity;
    sdata->mailboxid = mbentry->uniqueid;

    sdata->statusitems |= STATUS_MBENTRYITEMS;
}

HIDDEN void status_fill_mailbox(struct mailbox *mailbox, struct statusdata *sdata)
{
    assert(mailbox);
    assert(sdata);

    sdata->messages = mailbox->i.exists;
    sdata->uidnext = mailbox->i.last_uid+1;
    sdata->mboptions = mailbox->i.options;
    sdata->size = mailbox->i.quota_mailbox_used;
    sdata->createdmodseq = mailbox->i.createdmodseq;
    sdata->highestmodseq = mailbox->i.highestmodseq;

    // mbentry items are also available from an open mailbox
    sdata->uidvalidity = mailbox->i.uidvalidity;
    sdata->mailboxid = mailbox->uniqueid;

    sdata->statusitems |= STATUS_INDEXITEMS | STATUS_MBENTRYITEMS;
}

HIDDEN void status_fill_seen(const char *userid, struct statusdata *sdata,
                                  unsigned numrecent, unsigned numunseen)
{
    assert(userid);
    assert(sdata);

    // we need a matching parent record to exist for these values to be valid
    assert(sdata->statusitems & STATUS_HIGHESTMODSEQ);

    sdata->userid = userid;
    sdata->recent = numrecent;
    sdata->unseen = numunseen;

    sdata->statusitems |= STATUS_SEENITEMS;
}

static int status_load_mailbox(struct mailbox *mailbox, const char *userid,
                               unsigned statusitems, struct statusdata *sdata)
{
    status_fill_mailbox(mailbox, sdata);

    if ((statusitems & STATUS_SEENITEMS) && mailbox->i.exists) {
        unsigned numrecent = 0;
        unsigned numunseen = 0;
        /* Read \Seen state */
        struct seqset *seq = NULL;
        int internalseen = mailbox_internal_seen(mailbox, userid);
        unsigned recentuid;

        if (internalseen) {
            recentuid = mailbox->i.recentuid;
        } else {
            struct seen *seendb = NULL;
            struct seendata sd = SEENDATA_INITIALIZER;

            int r = seen_open(userid, SEEN_CREATE, &seendb);
            if (!r) r = seen_read(seendb, mailbox->uniqueid, &sd);
            seen_close(&seendb);
            if (r) return r;

            recentuid = sd.lastuid;
            seq = seqset_parse(sd.seenuids, NULL, recentuid);
            seen_freedata(&sd);
        }

        struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED);
        const message_t *msg;
        while ((msg = mailbox_iter_step(iter))) {
            const struct index_record *record = msg_record(msg);
            if (record->uid > recentuid)
                numrecent++;
            if (internalseen) {
                if (!(record->system_flags & FLAG_SEEN))
                    numunseen++;
            }
            else {
                if (!seqset_ismember(seq, record->uid))
                    numunseen++;
            }
        }
        mailbox_iter_done(&iter);
        seqset_free(seq);

        status_fill_seen(userid, sdata, numrecent, numunseen);
    }

    statuscache_invalidate(mailbox->name, sdata);

    return 0;
}

static int status_lookup_internal(const char *mboxname, const char *userid,
                                  unsigned statusitems, struct statusdata *sdata)
{
    struct mailbox *mailbox = NULL;
    int r = 0;

    /* Check status cache if possible */
    if (config_getswitch(IMAPOPT_STATUSCACHE)) {
        /* Do actual lookup of cache item. */
        r = statuscache_lookup(mboxname, userid, statusitems, sdata);

        /* Seen/recent status uses "push" invalidation events from
         * seen_db.c.   This avoids needing to open cyrus.header to get
         * the mailbox uniqueid to open the seen db and get the
         * unseen_mtime and recentuid.
         */

        if (!r) {
            syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'",
                   mboxname, userid, statusitems);
            return 0;
        }

        syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'",
               mboxname, userid, statusitems);
    }

    /* Missing or invalid cache entry */
    r = mailbox_open_irl(mboxname, &mailbox);
    if (r) return r;

    r = status_load_mailbox(mailbox, userid, statusitems, sdata);

    /* cache the new value while unlocking */
    if (!r) mailbox_unlock_index(mailbox, sdata);
    mailbox_close(&mailbox);

    return r;
}

EXPORTED int status_lookup_mbentry(const mbentry_t *mbentry, const char *userid,
                                  unsigned statusitems, struct statusdata *sdata)
{
    // check if we can get everything we need from the mbentry
    status_fill_mbentry(mbentry, sdata);
    if ((sdata->statusitems & statusitems) == statusitems)
        return 0;

    return status_lookup_internal(mbentry->name, userid, statusitems, sdata);
}

EXPORTED int status_lookup_mboxname(const char *mboxname, const char *userid,
                                    unsigned statusitems, struct statusdata *sdata)
{
    // we want an mbentry first, just in case we can get everything from there
    if (statusitems & STATUS_MAILBOXID) {
        mbentry_t *mbentry = NULL;
        int r = mboxlist_lookup_allow_all(mboxname, &mbentry, NULL);
        if (r) return r;
        r = status_lookup_mbentry(mbentry, userid, statusitems, sdata);
        mboxlist_entry_free(&mbentry);
        return r;
    }

    return status_lookup_internal(mboxname, userid, statusitems, sdata);
}


// this one has literally no smarts at all
EXPORTED int status_lookup_mbname(const mbname_t *mbname, const char *userid,
                                  unsigned statusitems, struct statusdata *sdata)
{
    return status_lookup_mboxname(mbname_intname(mbname), userid, statusitems, sdata);
}

/*
 * Performs a STATUS command on an open mailbox
 */
EXPORTED int status_lookup_mailbox(struct mailbox *mailbox, const char *userid,
                                   unsigned statusitems, struct statusdata *sdata)
{
    // check if we already have all the data we need (includes any possible mbentry)
    status_fill_mailbox(mailbox, sdata);
    if ((sdata->statusitems & statusitems) == statusitems)
        return 0;

    /* Check status cache if possible */
    if (config_getswitch(IMAPOPT_STATUSCACHE)) {
        /* Do actual lookup of cache item. */
        int r = statuscache_lookup(mailbox->name, userid, statusitems, sdata);

        /* Seen/recent status uses "push" invalidation events from
         * seen_db.c.   This avoids needing to open cyrus.header to get
         * the mailbox uniqueid to open the seen db and get the
         * unseen_mtime and recentuid.
         */

        if (!r) {
            syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'",
                   mailbox->name, userid, statusitems);
            return 0;
        }

        syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'",
               mailbox->name, userid, statusitems);
    }

    return status_load_mailbox(mailbox, userid, statusitems, sdata);
}