history/hisv6/hisv6.c

/*  $Id: hisv6.c 10305 2018-12-02 14:21:56Z iulius $
**
**  History v6 implementation against the history API.
**
**  Copyright (c) 2001, Thus plc
**
**  Redistribution and use of the source code in source and binary
**  forms, with or without modification, are permitted provided that
**  the following 3 conditions are met:
**
**  1. Redistributions of the source code must retain the above
**  copyright notice, this list of conditions and the disclaimer
**  set out below.
**
**  2. Redistributions of the source code in binary form must
**  reproduce the above copyright notice, this list of conditions
**  and the disclaimer set out below in the documentation and/or
**  other materials provided with the distribution.
**
**  3. Neither the name of the Thus plc nor the names of its
**  contributors may be used to endorse or promote products
**  derived from this software without specific prior written
**  permission from Thus plc.
**
**  Disclaimer:
**
**  "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
**  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
**  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
**  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE DIRECTORS
**  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
**  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
**  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
**  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
**  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
**  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
**  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
*/

#include "config.h"
#include "clibrary.h"
#include <fcntl.h>
#include <limits.h>
#include <errno.h>
#include "hisinterface.h"
#include "hisv6.h"
#include "hisv6-private.h"
#include "inn/dbz.h"
#include "inn/fdflag.h"
#include "inn/innconf.h"
#include "inn/timer.h"
#include "inn/qio.h"
#include "inn/sequence.h"
#include "inn/inndcomm.h"

/*
**  because we can only have one open dbz per process, we keep a
**  pointer to which of the current history structures owns it
*/
static struct hisv6 *hisv6_dbzowner;


/*
**  set error status to that indicated by s; doesn't copy the string,
**  assumes the caller did that for us
*/
static void
hisv6_seterror(struct hisv6 *h, const char *s)
{
    his_seterror(h->history, s);
}


/*
**  format line or offset into a string for error reporting
*/
static void
hisv6_errloc(char *s, size_t line, off_t offset)
{
    if (offset != -1) {
	/* really we want an autoconf test for %ll/%L/%I64, sigh */
        snprintf(s, HISV6_MAX_LOCATION, "@%lu", (unsigned long)offset);
    } else {
        snprintf(s, HISV6_MAX_LOCATION, ":%lu", (unsigned long)line);
    }
}


/*
**  split a history line into its constituent components; return a
**  bitmap indicating which components we're returning are valid (or
**  would be valid if a NULL pointer is passed for that component) or
**  -1 for error.  *error is set to a string which describes the
**  failure.
*/
static int
hisv6_splitline(const char *line, const char **error, HASH *hash,
		 time_t *arrived, time_t *posted, time_t *expires,
		 TOKEN *token)
{
    const char *p = line;
    char *end;
    unsigned long l;
    int r = 0;

    /* parse the [...] hash field */
    if (*p != '[') {
	*error = "`[' missing from history line";
	return -1;
    }
    ++p;
    if (hash)
	*hash = TextToHash(p);
    p += 32;
    if (*p != ']') {
	*error = "`]' missing from history line";
	return -1;
    }
    ++p;
    r |= HISV6_HAVE_HASH;
    if (*p != HISV6_FIELDSEP) {
	*error = "field separator missing from history line";
	return -1;
    }

    /* parse the arrived field */
    l = strtoul(p + 1, &end, 10);
    p = end;
    if (l == ULONG_MAX) {
	*error = "arrived timestamp out of range";
	return -1;
    }
    r |= HISV6_HAVE_ARRIVED;
    if (arrived)
	*arrived = (time_t)l;
    if (*p != HISV6_SUBFIELDSEP) {
	/* no expires or posted time */
	if (posted)
	    *posted = 0;
	if (expires)
	    *expires = 0;
    } else {
	/* parse out the expires field */
	++p;
	if (*p == HISV6_NOEXP) {
	    ++p;
	    if (expires)
		*expires = 0;
	} else {
	    l = strtoul(p, &end, 10);
            p = end;
	    if (l == ULONG_MAX) {
		*error = "expires timestamp out of range";
		return -1;
	    }
	    r |= HISV6_HAVE_EXPIRES;
	    if (expires)
		*expires = (time_t)l;
	}
	/* parse out the posted field */
	if (*p != HISV6_SUBFIELDSEP) {
	    /* no posted time */
	    if (posted)
		*posted = 0;
	} else {
	    ++p;
	    l = strtoul(p, &end, 10);
            p = end;
	    if (l == ULONG_MAX) {
		*error = "posted timestamp out of range";
		return -1;
	    }
	    r |= HISV6_HAVE_POSTED;
	    if (posted)
		*posted = (time_t)l;
	}
    }

    /* parse the token */
    if (*p == HISV6_FIELDSEP)
	++p;
    else if (*p != '\0') {
	*error = "field separator missing from history line";
	return -1;
    }
    /* IsToken false would imply a remembered line, or where someone's
     * used prunehistory */
    if (IsToken(p)) {
	r |= HISV6_HAVE_TOKEN;
	if (token)
	    *token = TextToToken(p);
    }
    return r;
}


/*
**  Given the time, now, return the time at which we should next check
**  the history file
*/
static unsigned long
hisv6_nextcheck(struct hisv6 *h, unsigned long now)
{
    return now + h->statinterval;
}


/*
**  close any dbz structures associated with h; we also manage the
**  single dbz instance voodoo
*/
static bool
hisv6_dbzclose(struct hisv6 *h)
{
    bool r = true;

    if (h == hisv6_dbzowner) {
	if (!hisv6_sync(h))
	    r = false;
	if (!dbzclose()) {
	    hisv6_seterror(h, concat("can't dbzclose ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    r = false;
	}
	hisv6_dbzowner = NULL;
    }
    return r;
}


/*
**  close an existing history structure, cleaning it to the point
**  where we can reopon without leaking resources
*/
static bool
hisv6_closefiles(struct hisv6 *h)
{
    bool r = true;

    if (!hisv6_dbzclose(h))
	r = false;

    if (h->readfd != -1) {
	if (close(h->readfd) != 0 && errno != EINTR) {
	    hisv6_seterror(h, concat("can't close history ",
				      h->histpath, " ",
				      strerror(errno),NULL));
	    r = false;
	}
	h->readfd = -1;
    }

    if (h->writefp != NULL) {
	if (ferror(h->writefp) || fflush(h->writefp) == EOF) {
	    hisv6_seterror(h, concat("error on history ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    r = false;
	}
	if (Fclose(h->writefp) == EOF) {
	    hisv6_seterror(h, concat("can't fclose history ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    r = false;
	}
	h->writefp = NULL;
        h->offset = 0;
    }

    h->nextcheck = 0;
    h->st.st_ino = (ino_t)-1;
    h->st.st_dev = (dev_t)-1;
    return r;
}


/*
**  Reopen (or open from fresh) a history structure; assumes the flags
**  & path are all set up, ready to roll. If we don't own the dbz, we
**  suppress the dbz code; this is needed during expiry (since the dbz
**  code doesn't yet understand multiple open contexts... yes its a
**  hack)
*/
static bool
hisv6_reopen(struct hisv6 *h)
{
    bool r = false;

    if (h->flags & HIS_RDWR) {
	const char *mode;

	if (h->flags & HIS_CREAT)
	    mode = "w";
	else
	    mode = "r+";
	if ((h->writefp = Fopen(h->histpath, mode, INND_HISTORY)) == NULL) {
	    hisv6_seterror(h, concat("can't fopen history ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    hisv6_closefiles(h);
	    goto fail;
	}
	if (fseeko(h->writefp, 0, SEEK_END) == -1) {
	    hisv6_seterror(h, concat("can't fseek to end of ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    hisv6_closefiles(h);
	    goto fail;
	}
        h->offset = ftello(h->writefp);
	if (h->offset == -1) {
	    hisv6_seterror(h, concat("can't ftello ", h->histpath, " ",
				      strerror(errno), NULL));
	    hisv6_closefiles(h);
	    goto fail;
	}
	fdflag_close_exec(fileno(h->writefp), true);
    }

    /* Open the history file for reading. */
    if ((h->readfd = open(h->histpath, O_RDONLY)) < 0) {
	hisv6_seterror(h, concat("can't open ", h->histpath, " ",
				  strerror(errno), NULL));
	hisv6_closefiles(h);
	goto fail;
    }
    fdflag_close_exec(h->readfd, true);

    /* if there's no current dbz owner, claim it here */
    if (hisv6_dbzowner == NULL) {
	hisv6_dbzowner = h;
    }

    /* During expiry we need two history structures in place, so we
       have to select which one gets the dbz file */
    if (h == hisv6_dbzowner) {
	dbzoptions opt;

	/* Open the DBZ file. */
	dbzgetoptions(&opt);

	/* HIS_INCORE usually means we're rebuilding from scratch, so
	   keep the whole lot in core until we flush */
	if (h->flags & HIS_INCORE) {
	    opt.writethrough = false;
	    opt.pag_incore = INCORE_MEM;
#ifndef	DO_TAGGED_HASH
	    opt.exists_incore = INCORE_MEM;
#endif
	} else {
	    opt.writethrough = true;
#ifdef	DO_TAGGED_HASH
	    opt.pag_incore = INCORE_MMAP;
#else
	    /*opt.pag_incore = INCORE_NO;*/
	    opt.pag_incore = (h->flags & HIS_MMAP) ? INCORE_MMAP : INCORE_NO;
	    opt.exists_incore = (h->flags & HIS_MMAP) ? INCORE_MMAP : INCORE_NO;

# if defined(MMAP_NEEDS_MSYNC) && INND_DBZINCORE == 1
	    /* Systems that have MMAP_NEEDS_MSYNC defined will have their
	       on-disk copies out of sync with the mmap'ed copies most of
	       the time.  So if innd is using INCORE_MMAP, then we force
	       everything else to use it, too (unless we're on NFS) */
	    if(!innconf->nfsreader) {
		opt.pag_incore = INCORE_MMAP;
		opt.exists_incore = INCORE_MMAP;
	    }
# endif
#endif
	}
	dbzsetoptions(opt);
	if (h->flags & HIS_CREAT) {
	    size_t npairs;

	    /* must only do this once! */
	    h->flags &= ~HIS_CREAT;
	    npairs = (h->npairs == -1) ? 0 : h->npairs;
	    if (!dbzfresh(h->histpath, dbzsize(npairs))) {
		hisv6_seterror(h, concat("can't dbzfresh ", h->histpath, " ",
					  strerror(errno), NULL));
		hisv6_closefiles(h);
		goto fail;
	    }
	} else if (!dbzinit(h->histpath)) {
	    hisv6_seterror(h, concat("can't dbzinit ", h->histpath, " ",
				      strerror(errno), NULL));
	    hisv6_closefiles(h);
	    goto fail;
	}
    }
    h->nextcheck = hisv6_nextcheck(h, TMRnow());
    r = true;
 fail:
    return r;
}


/*
** check if the history file has changed, if so rotate to the new
** history file. Returns false on failure (which is probably fatal as
** we'll have closed the files)
*/
static bool
hisv6_checkfiles(struct hisv6 *h)
{
    unsigned long t = TMRnow();

    if (h->statinterval == 0)
	return true;

    if (h->readfd == -1) {
	/* this can happen if a previous checkfiles() has failed to
	 * reopen the handles, but our caller hasn't realised... */
	hisv6_closefiles(h);
	if (!hisv6_reopen(h)) {
	    hisv6_closefiles(h);
	    return false;
	}
    }
    if (seq_lcompare(t, h->nextcheck) == 1) {
	struct stat st;

	if (stat(h->histpath, &st) == 0 &&
	    (st.st_ino != h->st.st_ino ||
	     st.st_dev != h->st.st_dev)) {
	    /* there's a possible race on the history file here... */
	    hisv6_closefiles(h);
	    if (!hisv6_reopen(h)) {
		hisv6_closefiles(h);
		return false;
	    }
	    h->st = st;
	}
	h->nextcheck = hisv6_nextcheck(h, t);
    }
    return true;
}


/*
**  dispose (and clean up) an existing history structure
*/
static bool
hisv6_dispose(struct hisv6 *h)
{
    bool r;

    r = hisv6_closefiles(h);
    if (h->histpath) {
	free(h->histpath);
	h->histpath = NULL;
    }

    free(h);
    return r;
}


/*
**  return a newly constructed, but empty, history structure
*/
static struct hisv6 *
hisv6_new(const char *path, int flags, struct history *history)
{
    struct hisv6 *h;

    h = xmalloc(sizeof *h);
    h->histpath = path ? xstrdup(path) : NULL;
    h->flags = flags;
    h->writefp = NULL;
    h->offset = 0;
    h->history = history;
    h->readfd = -1;
    h->nextcheck = 0;
    h->statinterval = 0;
    h->npairs = 0;
    h->dirty = 0;
    h->synccount = 0;
    h->st.st_ino = (ino_t)-1;
    h->st.st_dev = (dev_t)-1;
    return h;
}


/*
**  open the history database identified by path in mode flags
*/
void *
hisv6_open(const char *path, int flags, struct history *history)
{
    struct hisv6 *h;

    his_logger("HISsetup begin", S_HISsetup);

    h = hisv6_new(path, flags, history);
    if (path) {
	if (!hisv6_reopen(h)) {
	    hisv6_dispose(h);
	    h = NULL;
	}
    }
    his_logger("HISsetup end", S_HISsetup);
    return h;
}


/*
**  close and free a history handle
*/
bool
hisv6_close(void *history)
{
    struct hisv6 *h = history;
    bool r;

    his_logger("HISclose begin", S_HISclose);
    r = hisv6_dispose(h);
    his_logger("HISclose end", S_HISclose);
    return r;
}


/*
**  synchronise any outstanding history changes to disk
*/
bool
hisv6_sync(void *history)
{
    struct hisv6 *h = history;
    bool r = true;

    if (h->writefp != NULL) {
	his_logger("HISsync begin", S_HISsync);
	if (fflush(h->writefp) == EOF) {
	    hisv6_seterror(h, concat("error on history ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    r = false;
	}
	if (h->dirty && h == hisv6_dbzowner) {
	    if (!dbzsync()) {
		hisv6_seterror(h, concat("can't dbzsync ", h->histpath,
					  " ", strerror(errno), NULL));
		r = false;
	    } else {
		h->dirty = 0;
	    }
	}
	his_logger("HISsync end", S_HISsync);
    }
    return r;
}


/*
**  fetch the line associated with `hash' in the history database into
**  buf; buf must be at least HISV6_MAXLINE+1 bytes. `poff' is filled
**  with the offset of the line in the history file.
*/
static bool
hisv6_fetchline(struct hisv6 *h, const HASH *hash, char *buf, off_t *poff)
{
    off_t offset;
    bool r;

    if (h != hisv6_dbzowner) {
	hisv6_seterror(h, concat("dbz not open for this history file ",
				  h->histpath, NULL));
	return false;
    }
    if ((h->flags & (HIS_RDWR | HIS_INCORE)) == (HIS_RDWR | HIS_INCORE)) {
	/* need to fflush as we may be reading uncommitted data
	   written via writefp */
	if (fflush(h->writefp) == EOF) {
	    hisv6_seterror(h, concat("error on history ",
				      h->histpath, " ",
				      strerror(errno), NULL));
	    r = false;
	    goto fail;
	}
    }

    /* Get the seek value into the history file. */
    errno = 0;
    r = dbzfetch(*hash, &offset);
#ifdef ESTALE
    /* If your history is on NFS need to deal with stale NFS
     * handles */
    if (!r && errno == ESTALE) {
	hisv6_closefiles(h);
	if (!hisv6_reopen(h)) {
	    hisv6_closefiles(h);
	    r = false;
	    goto fail;
	}
    }
#endif
    if (r) {
	ssize_t n;

	do {
	    n = pread(h->readfd, buf, HISV6_MAXLINE, offset);
#ifdef ESTALE
	    if (n == -1 && errno == ESTALE) {
		hisv6_closefiles(h);
		if (!hisv6_reopen(h)) {
		    hisv6_closefiles(h);
		    r = false;
		    goto fail;
		}
	    }
#endif
	} while (n == -1 && errno == EINTR);
	if (n >= HISV6_MINLINE) {
	    char *p;

	    buf[n] = '\0';
	    p = strchr(buf, '\n');
	    if (!p) {
		char location[HISV6_MAX_LOCATION];

		hisv6_errloc(location, (size_t)-1, offset);
		hisv6_seterror(h,
				concat("can't locate end of line in history ",
				       h->histpath, location,
				       NULL));
		r = false;
	    } else {
		*p = '\0';
		*poff = offset;
		r = true;
	    }
	} else {
	    char location[HISV6_MAX_LOCATION];

	    hisv6_errloc(location, (size_t)-1, offset);
	    hisv6_seterror(h, concat("line too short in history ",
				      h->histpath, location,
				      NULL));
	    r = false;

	}
    } else {
	/* not found */
	r = false;
    }
 fail:
    return r;
}


/*
**  lookup up the entry `key' in the history database, returning
**  arrived, posted and expires (for those which aren't NULL
**  pointers), and any storage token associated with the entry.
**
**  If any of arrived, posted or expires aren't available, return zero
**  for that component.
*/
bool
hisv6_lookup(void *history, const char *key, time_t *arrived,
	     time_t *posted, time_t *expires, TOKEN *token)
{
    struct hisv6 *h = history;
    HASH messageid;
    bool r;
    off_t offset;
    char buf[HISV6_MAXLINE + 1];

    his_logger("HISfilesfor begin", S_HISfilesfor);
    hisv6_checkfiles(h);

    messageid = HashMessageID(key);
    r = hisv6_fetchline(h, &messageid, buf, &offset);
    if (r == true) {
	int status;
	const char *error;

	status = hisv6_splitline(buf, &error, NULL,
				  arrived, posted, expires, token);
	if (status < 0) {
	    char location[HISV6_MAX_LOCATION];

	    hisv6_errloc(location, (size_t)-1, offset);
	    hisv6_seterror(h, concat(error, " ",
				      h->histpath, location,
				      NULL));
	    r = false;
	} else {
	    /* if we have a token then we have the article */
	    r = !!(status & HISV6_HAVE_TOKEN);
	}
    }
    his_logger("HISfilesfor end", S_HISfilesfor);
    return r;
}


/*
**  check `key' has been seen in this history database
*/
bool
hisv6_check(void *history, const char *key)
{
    struct hisv6 *h = history;
    bool r;
    HASH hash;

    if (h != hisv6_dbzowner) {
	hisv6_seterror(h, concat("dbz not open for this history file ",
				  h->histpath, NULL));
	return false;
    }

    his_logger("HIShavearticle begin", S_HIShavearticle);
    hisv6_checkfiles(h);
    hash = HashMessageID(key);
    r = dbzexists(hash);
    his_logger("HIShavearticle end", S_HIShavearticle);
    return r;
}


/*
**  Format a history line.  s should hold at least HISV6_MAXLINE + 1
**  characters (to allow for the nul).  Returns the length of the data
**  written, 0 if there was some error or if the data was too long to write.
*/
static int
hisv6_formatline(char *s, const HASH *hash, time_t arrived,
		  time_t posted, time_t expires, const TOKEN *token)
{
    int i;
    const char *hashtext = HashToText(*hash);

    if (token == NULL) {
        /* Only a line to remember an article.  We keep its arrival
         * and posting time. */
        if (posted <= 0) {
            i = snprintf(s, HISV6_MAXLINE, "[%s]%c%lu%c%c\n",
                         hashtext, HISV6_FIELDSEP,
                         (unsigned long)arrived, HISV6_SUBFIELDSEP,
                         HISV6_NOEXP);
        } else {
            i = snprintf(s, HISV6_MAXLINE, "[%s]%c%lu%c%c%c%lu\n",
                         hashtext, HISV6_FIELDSEP,
                         (unsigned long)arrived, HISV6_SUBFIELDSEP,
                         HISV6_NOEXP, HISV6_SUBFIELDSEP,
                         (unsigned long)posted);
        }
    } else {
	const char *texttok;

	texttok = TokenToText(*token);
	if (expires <= 0) {
	    i = snprintf(s, HISV6_MAXLINE, "[%s]%c%lu%c%c%c%lu%c%s\n",
			 hashtext, HISV6_FIELDSEP,
			 (unsigned long)arrived, HISV6_SUBFIELDSEP,
			 HISV6_NOEXP, HISV6_SUBFIELDSEP,
			 (unsigned long)posted, HISV6_FIELDSEP,
			 texttok);
	} else {
	    i = snprintf(s, HISV6_MAXLINE, "[%s]%c%lu%c%lu%c%lu%c%s\n",
			 hashtext, HISV6_FIELDSEP,
			 (unsigned long)arrived, HISV6_SUBFIELDSEP,
			 (unsigned long)expires, HISV6_SUBFIELDSEP,
			 (unsigned long)posted, HISV6_FIELDSEP,
			 texttok);
	}
    }
    if (i < 0 || i >= HISV6_MAXLINE)
	return 0;
    return i;
}


/*
**  write the hash and offset to the dbz
*/
static bool
hisv6_writedbz(struct hisv6 *h, const HASH *hash, off_t offset)
{
    bool r;
    char location[HISV6_MAX_LOCATION];
    const char *error;

    /* store the offset in the database */
    switch (dbzstore(*hash, offset)) {
    case DBZSTORE_EXISTS:
	error = "dbzstore duplicate message-id ";
	/* not `false' so that we duplicate the pre-existing
	   behaviour */
	r = true;
	break;

    case DBZSTORE_ERROR:
	error = "dbzstore error ";
	r = false;
	break;

    default:
	error = NULL;
	r = true;
	break;
    }
    if (error) {
	hisv6_errloc(location, (size_t)-1, offset);
	hisv6_seterror(h, concat(error, h->histpath,
				  ":[", HashToText(*hash), "]",
				  location, " ", strerror(errno), NULL));
    }
    if (r && h->synccount != 0 && ++h->dirty >= h->synccount)
	r = hisv6_sync(h);

    return r;
}


/*
**  write a history entry, hash, with times arrived, posted and
**  expires, and storage token.
*/
static bool
hisv6_writeline(struct hisv6 *h, const HASH *hash, time_t arrived,
		time_t posted, time_t expires, const TOKEN *token)
{
    bool r;
    size_t i, length;
    char hisline[HISV6_MAXLINE + 1];
    char location[HISV6_MAX_LOCATION];

    if (h != hisv6_dbzowner) {
	hisv6_seterror(h, concat("dbz not open for this history file ",
				  h->histpath, NULL));
	return false;
    }

    if (!(h->flags & HIS_RDWR)) {
	hisv6_seterror(h, concat("history not open for writing ",
				  h->histpath, NULL));
	return false;
    }

    length = hisv6_formatline(hisline, hash, arrived, posted, expires, token);
    if (length == 0) {
	hisv6_seterror(h, concat("error formatting history line ",
				  h->histpath, NULL));
	return false;
    }

    i = fwrite(hisline, 1, length, h->writefp);

    /* If the write failed, the history line is now an orphan.  Attempt to
       rewind the write pointer to our offset to avoid leaving behind a
       partial write and desyncing the offset from our file position. */
    if (i < length ||
        (!(h->flags & HIS_INCORE) && fflush(h->writefp) == EOF)) {
	hisv6_errloc(location, (size_t)-1, h->offset);
	hisv6_seterror(h, concat("can't write history ", h->histpath,
				  location, " ", strerror(errno), NULL));
        if (fseeko(h->writefp, h->offset, SEEK_SET) == -1)
            h->offset += i;
	r = false;
	goto fail;
    }

    r = hisv6_writedbz(h, hash, h->offset);
    h->offset += length;     /* increment regardless of error from writedbz */
 fail:
    return r;
}


/*
**  write a history entry, key, with times arrived, posted and
**  expires, and storage token.
*/
bool
hisv6_write(void *history, const char *key, time_t arrived,
	    time_t posted, time_t expires, const TOKEN *token)
{
    struct hisv6 *h = history;
    HASH hash;
    bool r;

    his_logger("HISwrite begin", S_HISwrite);
    hash = HashMessageID(key);
    r = hisv6_writeline(h, &hash, arrived, posted, expires, token);
    his_logger("HISwrite end", S_HISwrite);
    return r;
}


/*
**  Remember a history entry, key, with arrival time, and also
**  posting time if known.
*/
bool
hisv6_remember(void *history, const char *key, time_t arrived, time_t posted)
{
    struct hisv6 *h = history;
    HASH hash;
    bool r;

    his_logger("HISwrite begin", S_HISwrite);
    hash = HashMessageID(key);
    r = hisv6_writeline(h, &hash, arrived, posted, 0, NULL);
    his_logger("HISwrite end", S_HISwrite);
    return r;
}


/*
**  replace an existing history entry, `key', with times arrived,
**  posted and expires, and (optionally) storage token `token'. The
**  new history line must fit in the space allocated for the old one -
**  if it had previously just been HISremember()ed you'll almost
**  certainly lose.
*/
bool
hisv6_replace(void *history, const char *key, time_t arrived,
	      time_t posted, time_t expires, const TOKEN *token)
{
    struct hisv6 *h = history;
    HASH hash;
    bool r;
    off_t offset;
    char old[HISV6_MAXLINE + 1];

    if (!(h->flags & HIS_RDWR)) {
	hisv6_seterror(h, concat("history not open for writing ",
				  h->histpath, NULL));
	return false;
    }

    hash = HashMessageID(key);
    r = hisv6_fetchline(h, &hash, old, &offset);
    if (r == true) {
	char new[HISV6_MAXLINE + 1];

	if (hisv6_formatline(new, &hash, arrived, posted, expires,
                             token) == 0) {
 	    hisv6_seterror(h, concat("error formatting history line ",
				      h->histpath, NULL));
	    r = false;
	} else {
	    size_t oldlen, newlen;

	    oldlen = strlen(old);
	    newlen = strlen(new);
	    if (new[newlen - 1] == '\n')
                newlen--;
	    if (newlen > oldlen) {
		hisv6_seterror(h, concat("new history line too long ",
					  h->histpath, NULL));
		r = false;
	    } else {
		ssize_t n;

		/* space fill any excess in the tail of new */
		memset(new + newlen, ' ', oldlen - newlen);

		do {
		    n = pwrite(fileno(h->writefp), new, oldlen, offset);
		} while (n == -1 && errno == EINTR);
		if ((size_t) n != oldlen) {
		    char location[HISV6_MAX_LOCATION];

		    hisv6_errloc(location, (size_t)-1, offset);
		    hisv6_seterror(h, concat("can't write history ",
					      h->histpath, location, " ",
					      strerror(errno), NULL));
		    r = false;
		}
	    }
	}
    }
    return r;
}


/*
**  traverse a history database, passing the pieces through a
**  callback; note that we have more parameters in the callback than
**  the public interface, we add the internal history struct and the
**  message hash so we can use those if we need them. If the callback
**  returns false we abort the traversal.
**/
static bool
hisv6_traverse(struct hisv6 *h, struct hisv6_walkstate *cookie,
	       const char *reason,
	       bool (*callback)(struct hisv6 *, void *, const HASH *hash,
				time_t, time_t, time_t,
				const TOKEN *))
{
    bool r = false;
    QIOSTATE *qp;
    void *p;
    size_t line;
    char location[HISV6_MAX_LOCATION];

    if ((qp = QIOopen(h->histpath)) == NULL) {
	hisv6_seterror(h, concat("can't QIOopen history file ",
				  h->histpath, strerror(errno), NULL));
	return false;
    }

    line = 1;
    /* we come back to again after we hit EOF for the first time, when
       we pause the server & clean up any lines which sneak through in
       the interim */
 again:
    while ((p = QIOread(qp)) != NULL) {
	time_t arrived, posted, expires;
	int status;
	TOKEN token;
	HASH hash;
	const char *error;

	status = hisv6_splitline(p, &error, &hash,
				  &arrived, &posted, &expires, &token);
	if (status > 0) {
	    r = (*callback)(h, cookie, &hash, arrived, posted, expires,
			    (status & HISV6_HAVE_TOKEN) ? &token : NULL);
	    if (r == false)
		hisv6_seterror(h, concat("callback failed ",
					  h->histpath, NULL));
	} else {
	    hisv6_errloc(location, line, (off_t)-1);
	    hisv6_seterror(h, concat(error, " ", h->histpath, location,
				      NULL));
	    /* if we're not ignoring errors set the status */
	    if (!cookie->ignore)
		r = false;
	}
	if (r == false)
	    goto fail;
	++line;
    }

    if (p == NULL) {
	/* read or line-format error? */
	if (QIOerror(qp) || QIOtoolong(qp)) {
	    hisv6_errloc(location, line, (off_t)-1);
	    if (QIOtoolong(qp)) {
		hisv6_seterror(h, concat("line too long ",
					 h->histpath, location, NULL));
		/* if we're not ignoring errors set the status */
		if (!cookie->ignore)
		    r = false;
	    } else {
		hisv6_seterror(h, concat("can't read line ",
					 h->histpath, location, " ",
					 strerror(errno), NULL));
		r = false;
	    }
	    if (r == false)
		goto fail;
	}

	/* must have been EOF, pause the server & clean up any
	 * stragglers */
	if (reason && !cookie->paused) {
	    if (ICCpause(reason) != 0) {
		hisv6_seterror(h, concat("can't pause server ",
					  h->histpath, strerror(errno), NULL));
		r = false;
		goto fail;
	    }
	    cookie->paused = true;
	    goto again;
	}
    }
 fail:
    QIOclose(qp);
    return r;
}


/*
**  internal callback used during hisv6_traverse; we just pass on the
**  parameters the user callback expects
**/
static bool
hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash UNUSED,
		 time_t arrived, time_t posted, time_t expires,
		 const TOKEN *token)
{
    struct hisv6_walkstate *hiscookie = cookie;

    return (*hiscookie->cb.walk)(hiscookie->cookie,
				 arrived, posted, expires,
				 token);
}


/*
**  history API interface to the database traversal routine
*/
bool
hisv6_walk(void *history, const char *reason, void *cookie,
	   bool (*callback)(void *, time_t, time_t, time_t,
			    const TOKEN *))
{
    struct hisv6 *h = history;
    struct hisv6_walkstate hiscookie;
    bool r;

    /* our internal walk routine passes too many parameters, so add a
       wrapper */
    hiscookie.cb.walk = callback;
    hiscookie.cookie = cookie;
    hiscookie.new = NULL;
    hiscookie.paused = false;
    hiscookie.ignore = false;

    r = hisv6_traverse(h, &hiscookie, reason, hisv6_traversecb);

    return r;
}


/*
**  internal callback used during expire
**/
static bool
hisv6_expirecb(struct hisv6 *h, void *cookie, const HASH *hash,
		time_t arrived, time_t posted, time_t expires,
		const TOKEN *token)
{
    struct hisv6_walkstate *hiscookie = cookie;
    bool r = true;

    /* check if we've seen this message id already */
    if (hiscookie->new && dbzexists(*hash)) {
	/* continue after duplicates, it's serious, but not fatal */
	hisv6_seterror(h, concat("duplicate message-id [",
				 HashToText(*hash), "] in history ",
				 hiscookie->new->histpath, NULL));
    } else {
	struct hisv6_walkstate *hiscookie = cookie;
	TOKEN ltoken, *t;

	/* if we have a token pass it to the discrimination function */
	if (token) {
	    bool keep;

	    /* make a local copy of the token so the callback can
	     * modify it */
	    ltoken = *token;
	    t = &ltoken;
	    keep = (*hiscookie->cb.expire)(hiscookie->cookie,
					   arrived, posted, expires,
					   t);
	    /* If the callback returns true, we should keep the
	     * token for the time being, else we just remember
	     * it. */
	    if (keep == false) {
		t = NULL;
		expires = 0;
	    }
	} else {
	    t = NULL;
	}
        /* When t is NULL (no token), the message-ID is removed from
         * history when the posting time of the article is older than
         * threshold, as set by the /remember/ line in expire.ctl.
         * We keep the check for the arrival time because some entries
         * might not have one. */
	if (hiscookie->new &&
	    (t != NULL || posted >= hiscookie->threshold
             || (posted <= 0 && arrived >= hiscookie->threshold))) {
	    r = hisv6_writeline(hiscookie->new, hash,
				 arrived, posted, expires, t);
	}
    }
    return r;
}


/*
**  unlink files associated with the history structure h
*/
static bool
hisv6_unlink(struct hisv6 *h)
{
    bool r = true;
    char *p;

#ifdef DO_TAGGED_HASH
    p = concat(h->histpath, ".pag", NULL);
    r = (unlink(p) == 0) && r;
    free(p);
#else
    p = concat(h->histpath, ".index", NULL);
    r = (unlink(p) == 0) && r;
    free(p);

    p = concat(h->histpath, ".hash", NULL);
    r = (unlink(p) == 0) && r;
    free(p);
#endif

    p = concat(h->histpath, ".dir", NULL);
    r = (unlink(p) == 0) && r;
    free(p);

    r = (unlink(h->histpath) == 0) && r;
    return r;
}


/*
**  rename files associated with hold to hnew
*/
static bool
hisv6_rename(struct hisv6 *hold, struct hisv6 *hnew)
{
    bool r = true;
    char *old, *new;

#ifdef DO_TAGGED_HASH
    old = concat(hold->histpath, ".pag", NULL);
    new = concat(hnew->histpath, ".pag", NULL);
    r = (rename(old, new) == 0) && r;
    free(old);
    free(new);
#else
    old = concat(hold->histpath, ".index", NULL);
    new = concat(hnew->histpath, ".index", NULL);
    r = (rename(old, new) == 0) && r;
    free(old);
    free(new);

    old = concat(hold->histpath, ".hash", NULL);
    new = concat(hnew->histpath, ".hash", NULL);
    r = (rename(old, new) == 0) && r;
    free(old);
    free(new);
#endif

    old = concat(hold->histpath, ".dir", NULL);
    new = concat(hnew->histpath, ".dir", NULL);
    r = (rename(old, new) == 0) && r;
    free(old);
    free(new);

    r = (rename(hold->histpath, hnew->histpath) == 0) && r;
    return r;
}


/*
**  expire the history database, history.
*/
bool
hisv6_expire(void *history, const char *path, const char *reason,
	     bool writing, void *cookie, time_t threshold,
	     bool (*exists)(void *, time_t, time_t, time_t, TOKEN *))
{
    struct hisv6 *h = history, *hnew = NULL;
    char *nhistory = NULL;
    dbzoptions opt;
    bool r;
    struct hisv6_walkstate hiscookie;

    /* this flag is always tested in the fail clause, so initialise it
       now */
    hiscookie.paused = false;

    /* during expire we ignore errors whilst reading the history file
     * so any errors in it get fixed automagically */
    hiscookie.ignore = true;

    if (writing && (h->flags & HIS_RDWR)) {
	hisv6_seterror(h, concat("can't expire from read/write history ",
				 h->histpath, NULL));
	r = false;
	goto fail;
    }

    if (writing) {
	/* form base name for new history file */
	if (path != NULL) {
	    nhistory = concat(path, ".n", NULL);
	} else {
	    nhistory = concat(h->histpath, ".n", NULL);
	}

	hnew = hisv6_new(nhistory, HIS_CREAT | HIS_RDWR | HIS_INCORE,
                         h->history);
	if (!hisv6_reopen(hnew)) {
	    hisv6_dispose(hnew);
	    hnew = NULL;
	    r = false;
	    goto fail;
	}

	/* this is icky... we can only have one dbz open at a time; we
	   really want to make dbz take a state structure. For now we'll
	   just close the existing one and create our new one they way we
	   need it */
	if (!hisv6_dbzclose(h)) {
	    r = false;
	    goto fail;
	}

	dbzgetoptions(&opt);
	opt.writethrough = false;
	opt.pag_incore = INCORE_MEM;
#ifndef	DO_TAGGED_HASH
	opt.exists_incore = INCORE_MEM;
#endif
	dbzsetoptions(opt);

	if (h->npairs == 0) {
	    if (!dbzagain(hnew->histpath, h->histpath)) {
		hisv6_seterror(h, concat("can't dbzagain ",
					 hnew->histpath, ":", h->histpath,
					 strerror(errno), NULL));
		r = false;
		goto fail;
	    }
	} else {
	    size_t npairs;

	    npairs = (h->npairs == -1) ? 0 : h->npairs;
	    if (!dbzfresh(hnew->histpath, dbzsize(npairs))) {
		hisv6_seterror(h, concat("can't dbzfresh ",
					 hnew->histpath, ":", h->histpath,
					 strerror(errno), NULL));
		r = false;
		goto fail;
	    }
	}
	hisv6_dbzowner = hnew;
    }

    /* set up the callback handler */
    hiscookie.cb.expire = exists;
    hiscookie.cookie = cookie;
    hiscookie.new = hnew;
    hiscookie.threshold = threshold;
    r = hisv6_traverse(h, &hiscookie, reason, hisv6_expirecb);

 fail:
    if (writing) {
	if (hnew && !hisv6_closefiles(hnew)) {
	    /* error will already have been set */
	    r = false;
	}

	/* reopen will synchronise the dbz stuff for us */
	if (!hisv6_closefiles(h)) {
	    /* error will already have been set */
	    r = false;
	}

	if (r) {
	    /* if the new path was explicitly specified don't move the
	       files around, our caller is planning to do it out of
	       band */
	    if (path == NULL) {
		/* unlink the old files */
		r = hisv6_unlink(h);

		if (r) {
		    r = hisv6_rename(hnew, h);
		}
	    }
	} else if (hnew) {
	    /* something went pear shaped, unlink the new files */
	    hisv6_unlink(hnew);
	}

	/* re-enable dbz on the old history file */
	if (!hisv6_reopen(h)) {
	    hisv6_closefiles(h);
	}
    }

    if (hnew && !hisv6_dispose(hnew))
	r = false;
    if (nhistory && nhistory != path)
	free(nhistory);
    if (r == false && hiscookie.paused)
	ICCgo(reason);
    return r;
}


/*
**  control interface
*/
bool
hisv6_ctl(void *history, int selector, void *val)
{
    struct hisv6 *h = history;
    bool r = true;

    switch (selector) {
    case HISCTLG_PATH:
	*(char **)val = h->histpath;
	break;

    case HISCTLS_PATH:
	if (h->histpath) {
	    hisv6_seterror(h, concat("path already set in handle", NULL));
	    r = false;
	} else {
	    h->histpath = xstrdup((char *)val);
	    if (!hisv6_reopen(h)) {
		free(h->histpath);
		h->histpath = NULL;
		r = false;
	    }
	}
	break;

    case HISCTLS_STATINTERVAL:
	h->statinterval = *(time_t *)val * 1000;
	break;

    case HISCTLS_SYNCCOUNT:
	h->synccount = *(size_t *)val;
	break;

    case HISCTLS_NPAIRS:
	h->npairs = (ssize_t)*(size_t *)val;
	break;

    case HISCTLS_IGNOREOLD:
	if (h->npairs == 0 && *(bool *)val) {
	    h->npairs = -1;
	} else if (h->npairs == -1 && !*(bool *)val) {
	    h->npairs = 0;
	}
	break;

    default:
	/* deliberately doesn't call hisv6_seterror as we don't want
	 * to spam the error log if someone's passing in stuff which
	 * would be relevant to a different history manager */
	r = false;
	break;
    }
    return r;
}