xref: /dports/news/leafnode/leafnode-1.11.12/fetchnews.c

/*
fetchnews -- post articles to and get news from upstream server(s)

Written by Arnt Gulbrandsen <agulbra@troll.no> and copyright 1995
Troll Tech AS, Postboks 6133 Etterstad, 0602 Oslo, Norway, fax +47
22646949.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>
and Randolf Skerka <Randolf.Skerka@gmx.de>.
Copyright of the modifications 1997.
Modified by Kent Robotti <robotti@erols.com>. Copyright of the
modifications 1998.
Modified by Markus Enzenberger <enz@cip.physik.uni-muenchen.de>.
Copyright of the modifications 1998.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>.
Copyright of the modifications 1998, 1999.
Modified by Ralf Wildenhues <ralf.wildenhues@gmx.de>.
Copyright of the modifications 2002.
Modified by Jonathan Larmour <jifl@jifvik.org>.
Copyright of the modifications 2002.
Modified by Richard van der Hoff <richard@rvanderhoff.org.uk>
Copyright of the modifications 2002.
Enhanced and modified by Matthias Andree <matthias.andree@gmx.de>.
Copyright of the modifications 2000 - 2006.

See file COPYING for restrictions on the use of this software.
*/

#include "leafnode.h"
#include "fetchnews.h"
#include "mastring.h"
#include "ln_log.h"
#include "mysigact.h"

#include <sys/types.h>
#include <ctype.h>
#include "system.h"
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <setjmp.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <syslog.h>
#include <sys/resource.h>
#include <unistd.h>
#include <utime.h>
#include <sys/wait.h>

#include "groupselect.h"

int verbose = 0;
int debug = 0;

static time_t now;

/* Variables set by command-line options which are specific for fetch */
static unsigned long extraarticles = 0;
static int usesupplement = 1;
static int postonly = 0;	/* if 1, don't read files from upstream */
static int noexpire = 0;	/* if 1, don't automatically unsubscribe */
static int forceactive = 0;	/* if 1, reread complete active file */

static sigjmp_buf jmpbuffer;
static volatile sig_atomic_t canjump;

static int age( /*@null@*/ const char *date);
static int postarticles(const struct server *current_server);

static void
ignore_answer(FILE * f)
{
    char *l;
    while (((l = mgetaline(f)) != NULL) && strcmp(l, "."));
}

static void
sig_int(int signo)
{
    if (canjump == 0)
	return;		/* ignore unexpected signals */
    if (signo == SIGINT || signo == SIGTERM) {
	canjump = 0;
	alarm(0);
	siglongjmp(jmpbuffer, signo);
    }
}

static void
usage(void)
{
    fprintf(stderr, "Usage: fetchnews [-q] [-v] [-x #] [-l] [-n] [-f] [-P] [-w]\n"
	    "  -q: quiet, suppress some warnings, cancels -v\n"
	    "  -v: more verbose (may be repeated), cancels -q\n"
	    "  -x: check for # extra articles in each group\n"
	    "  -l: do not use supplementary servers\n"
	    "  -n: do not automatically expire unread groups\n"
	    "  -f: force reload of groupinfo file\n"
	    "  -P: only post outgoing articles, don't fetch any\n"
	    "  -w: wait, run XOVER updater in foreground\n");
}

/**
 * check whether any of the newsgroups is on server
 * return TRUE if yes, FALSE otherwise
 */
static int
isgrouponserver(const struct server *current_server,
	char *newsgroups /** string will be destroyed! */)
{
    char *p, *q;
    int retval;

    if (!newsgroups)
	return FALSE;

    retval = FALSE;
    p = newsgroups;
    do {
	q = strchr(p, ',');
	if (q)
	    *q++ = '\0';
	switch (gs_match(current_server->group_pcre, p)) {
	    case 1:
		xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", p);
		putaline();
		if (nntpreply(current_server) == 211) {
		    if (debug > 1)
			syslog(LOG_DEBUG, "%s is matched by only_groups_pcre and is on server", p);
		    retval = TRUE;
		}
		break;
	    case 0:
		if (debug > 1)
		    syslog(LOG_DEBUG, "%s not matched by only_group_pcre", p);
		if (current_server->only_groups_match_all) {
		    if (debug > 1)
			syslog(LOG_DEBUG, "not posting article to this server, "
				"only_groups_match_all is set");
		    return FALSE;
		}
		break;
	    default:
		break;
	}
	p = q;
	if (p)
	    while (*p && isspace((unsigned char)*p))
		p++;
    } while (q);

    return retval;
}

/*
 * check whether message-id is on server - msgid is expected
 * to contain the angle brackets, i. e. <id@example.org>
 * return 1 if yes, 0 if no, -1 for error
 */
static int
ismsgidonserver(const struct server *current_server, char *msgid)
{
    int r;
    if (!msgid)
	return 0;
    xsnprintf(lineout, SIZE_lineout, "%s %s\r\n",
	      stat_is_evil ? "HEAD" : "STAT", msgid);
    putaline();
    r = nntpreply(current_server);
    if (r == 498 && lastreply() == NULL)
	return -1;
    if (r >= 220 && r <= 223) {
	if (stat_is_evil)
	    ignore_answer(nntpin);
	return 1;
    } else
	return 0;
}

int
age( /*@null@*/ const char *date)
{
    char monthname[4]; /* RATS: ignore */
    int month;
    int year;
    int day;
    const char *d;
    time_t tmp;
    struct tm time_struct;

    if (!date)
	return 1000;		/* large number: OLD */
    d = date;
    if (!(strncasecmp(d, "date:", 5)))
	d += 5;
    while (isspace((unsigned char)*d))
	d++;

    if (isalpha((unsigned char)*d)) {
	while (*d && !isspace((unsigned char)*d))	/* skip "Mon" or "Tuesday," */
	    d++;
    }

    /* RFC 822 says we have 1*LWSP-char between tokens */
    while (isspace((unsigned char)*d))
	d++;

    /* parsing with sscanf leads to crashes */
    day = atoi(d);
    while (isdigit((unsigned char)*d) || isspace((unsigned char)*d))
	d++;
    if (!isalpha((unsigned char)*d)) {
	syslog(LOG_INFO, "Unable to parse %s", date);
	return 1003;
    }
    monthname[0] = *d++;
    monthname[1] = *d++;
    monthname[2] = *d++;
    monthname[3] = '\0';
    if (strlen(monthname) != 3) {
	syslog(LOG_INFO, "Unable to parse month in %s", date);
	return 1004;
    }
    while (isalpha((unsigned char)*d))
	d++;
    while (isspace((unsigned char)*d))
	d++;
    year = atoi(d);

    if ((year < 1970) && (year > 99)) {
	syslog(LOG_INFO, "Unable to parse year in %s", date);
	return 1005;
    } else if (!(day > 0 && day < 32)) {
	syslog(LOG_INFO, "Unable to parse day in %s", date);
	return 1006;
    } else {
	if (!strcasecmp(monthname, "jan"))
	    month = 0;
	else if (!strcasecmp(monthname, "feb"))
	    month = 1;
	else if (!strcasecmp(monthname, "mar"))
	    month = 2;
	else if (!strcasecmp(monthname, "apr"))
	    month = 3;
	else if (!strcasecmp(monthname, "may"))
	    month = 4;
	else if (!strcasecmp(monthname, "jun"))
	    month = 5;
	else if (!strcasecmp(monthname, "jul"))
	    month = 6;
	else if (!strcasecmp(monthname, "aug"))
	    month = 7;
	else if (!strcasecmp(monthname, "sep"))
	    month = 8;
	else if (!strcasecmp(monthname, "oct"))
	    month = 9;
	else if (!strcasecmp(monthname, "nov"))
	    month = 10;
	else if (!strcasecmp(monthname, "dec"))
	    month = 11;
	else {
	    syslog(LOG_INFO, "Unable to parse %s", date);
	    return 1001;
	}
	if (year < 70)		/* years 2000-2069 in two-digit form */
	    year += 100;
	else if (year > 1970)	/* years > 1970 in four-digit form */
	    year -= 1900;

	memset(&time_struct, 0, sizeof(time_struct));
	time_struct.tm_sec = 0;
	time_struct.tm_min = 0;
	time_struct.tm_hour = 0;
	time_struct.tm_mday = day;
	time_struct.tm_mon = month;
	time_struct.tm_year = year;
	time_struct.tm_isdst = 0;

	tmp = mktime(&time_struct);

	if (tmp == -1)
	    return 1002;

	return ((now - tmp) / SECONDS_PER_DAY);
    }
}

/*
 * Get body of a single message of which the header has already been
 * downloaded and append it to the file with the header.
 * Returns 0 if file could not be retrieved, 1 otherwise.
 */
static int
getbody_insitu(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
    const char *c;
    int rc = 0;
    char *l;
    char *messageid;
    FILE *f;
    char s[SIZE_s+1];
    off_t pos;

    if (!chdirgroup(group->name, FALSE))
	return 0;

    /* extract message-id: header */
    xsnprintf(s, SIZE_s, "%lu", id);
    messageid = getheader(s, "Message-ID:");
    if (!messageid)
	return 0;

    /* check whether we can retrieve the body */
    if (verbose > 2)
	printf("%s: BODY %s\n", group->name, messageid);
    xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
    putaline();

    if (nntpreply(current_server) != 222) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"%s: Retrieving body %s failed. No response",
	       group->name, messageid);
	rc = 0;
	goto getbody_bail;
    }

    xsnprintf(s, SIZE_s, "%lu", id);
    c = lookup(messageid);
    if (!(f = fopen(c, "a"))) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"%s: cannot open %s for appending", group->name, c);
	rc = 0;
	goto getbody_bail;
    }
    pos = ftell(f);
    fputc('\n', f); /* blank line -- separate header and body */

    debug--;
    while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
	if (*l == '.')
	    ++l;
	fputs(l, f);
	fputc('\n', f);
    }
    debug = debugmode;

    if (!l) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
	       "Transmission interrupted.", group->name, messageid);
	ftruncate(fileno(f), pos);
	fclose(f);
	rc = 0;
	goto getbody_bail;
    }

    /* abort when disk is full */
    if (fclose(f) && errno == ENOSPC) {
	truncate(s, pos);
	raise(SIGINT);
	return 0;
    }

    rc = 1;

  getbody_bail:
    if (messageid)
	free(messageid);
    return rc;
}

static int
getbody_newno(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
    char *c;
    const char *cc;
    int rc = 0;
    char *l;
    char *p, *q;
    char *messageid;
    char *newsgroups;		/* I hope this is enough */
    char *xref;
    FILE *f, *g;
    char s[SIZE_s+1];

    if (!chdirgroup(group->name, FALSE))
	return 0;

    /* extract message-id: and xref: headers */
    xsnprintf(s, SIZE_s, "%lu", id);
    if (!(f = fopen(s, "r"))) {
	syslog(LOG_INFO, "%s: cannot open %s for reading -- possibly expired",
	       group->name, s);
	return 1;		/* pretend to have read file successfully so that
				   it is purged from the list */
    }
    messageid = NULL;
    newsgroups = NULL;
    xref = NULL;
    debug--;
    while ((l = getaline(f)) != NULL) {
	if (!newsgroups && !strncmp(l, "Newsgroups:", 11)) {
	    p = l+11;
	    SKIPLWS(p);
	    newsgroups = critstrdup(p, "getbody");
	}
	if (!messageid && !strncmp(l, "Message-ID:", 11)) {
	    p = l+11;
	    SKIPLWS(p);
	    messageid = critstrdup(p, "getbody");
	}
	if (!xref && !strncmp(l, "Xref:", 5)) {
	    p = l + 5;
	    SKIPLWS(p);
	    xref = critstrdup(p, "getbody");
	}
    }
    debug = debugmode;
    fclose(f);

    if (!messageid || !*messageid) {
	/* haven't found Message-ID - cannot fetch */
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"%s: Retrieving body %lu failed, incomplete local header (MID).",
		group->name, id);
	rc = 1; /* pretend success, to purge the number from list -
		   retrying is pointless */
	goto getbody_bail;
    }

    /* check whether we can retrieve the body */
    if (verbose > 2)
	printf("%s: BODY %s\n", group->name, messageid);
    xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
    putaline();

    if (nntpreply(current_server) != 222) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. No response",
	       group->name, messageid);
	rc = 0;
	goto getbody_bail;
    }

    xsnprintf(s, SIZE_s, "%lu", id);
    cc = lookup(messageid);
    log_unlink(cc, 0);			/* make space for new file */

    if (!(f = fopen(cc, "w"))) {
	ln_log(LNLOG_SERR, LNLOG_CGROUP, "%s: cannot open %s for writing", group->name, cc);
	link(s, cc);		/* if we can't open new file restore old one */
	rc = 0;
	goto getbody_bail;
    }

    /* copy all headers except Xref: into new file */
    g = fopen(s, "r");
    if (!g) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: open %s failed", group->name, s);
	fclose(f);
	rc = 0;
	goto getbody_bail;
    }
    debug--;
    while ((l = getaline(g)) != NULL) {
	/* skip xref: headers */
	if (strncasecmp(l, "Xref:", 5) != 0)
	    fprintf(f, "%s\n", l);
    }
    debug = debugmode;
    fclose(g);

    /* create a whole bunch of new hardlinks */
    store(cc, f, newsgroups, messageid);

    /* retrieve body */
    fprintf(f, "\n");		/* Empty line between header and body. */
    debug--;
    while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
	if (*l == '.')
	    ++l;
	fputs(l, f);
	fputc('\n', f);
    }
    debug = debugmode;
    if (!l) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
	       "Transmission interrupted.", group->name, messageid);
	fprintf(f, "\n\t[ Leafnode: ]\n"
		"\t[ An error occured while " "retrieving this message. ]\n");
	fclose(f);
	rc = 0;
	goto getbody_bail;
    }
    /* abort when disk is full */
    if (fclose(f) && errno == ENOSPC)
	raise(SIGINT);

    /* Remove old article files since we don't need them anymore.
       This is done by evaluating the old Xref: header.
     */
    if (!xref) {
	/* no Xref: header --> make a fake one */
	xref = critmalloc(50 + strlen(fqdn) + strlen(group->name), "getbody");
	sprintf(xref, "%s %s:%lu", fqdn, group->name, id);	/* RATS: ignore */
    }

    if (debugmode)
	syslog(LOG_DEBUG, "xref: %s", xref);
    c = strchr(xref, ' ');
#ifdef __LCLINT__
    assert(c != NULL);		/* we know c != NULL */
#endif				/* __LCLINT__ */
    while ((c++) && (*c) && (q = strchr(c, ':')) != NULL) {
	*q++ = '\0';
	if ((p = strchr(q, ' ')) != NULL)
	    *p = '\0';

	/* c points to the newsgroup, q to the article number */
	if (!chdirgroup(c, FALSE)) {
	    return 0;
	}
	if (unlink(q))
	    syslog(LOG_NOTICE,
		   "retrieved body, but unlinking headers-only file %s/%s failed",
		   c, q);
	else if (debugmode)
	    syslog(LOG_DEBUG,
		   "retrieved body, now unlinking headers-only file %s/%s", c,
		   q);

	c = strchr(q, ' ');
    }
    rc = 1;
  getbody_bail:
    if (xref)
	free(xref);
    if (messageid)
	free(messageid);
    if (newsgroups)
	free(newsgroups);
    return rc;
}

static int
getbody(const struct server *cs, struct newsgroup *group, unsigned long id) {
    static int (*func)(const struct server *, struct newsgroup *, unsigned long);

    if (!func) {
	func = db_situ ? getbody_insitu : getbody_newno;
    }

    return func(cs, group, id);
}

/*
 * Get bodies of messages that have marked for download.
 * The group must already be selected at the remote server and
 * the current directory must be the one of the group.
 */
static void
getmarked(const struct server *current_server, struct newsgroup *group)
{
    int n, i;
    int had_bodies = 0;
    FILE *f;
    mastr *filename = mastr_new(PATH_MAX);
    unsigned long id[BODY_DOWNLOAD_LIMIT]; /* RATS: ignore */
    char *t;

    /* #1 read interesting.groups file */
    n = 0;
    mastr_vcat(filename, spooldir, "/interesting.groups/", group->name, NULL);
    if (!(f = fopen(mastr_str(filename), "r")))
	ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for reading", mastr_str(filename));
    else {
	struct stat st;
	if (fstat(fileno(f), &st) == 0 && st.st_size > 0) {
	    had_bodies = 1;
	    if (verbose)
		printf("%s: getting bodies of marked messages...\n",
			group->name);
	    while ((t = getaline(f)) && n < BODY_DOWNLOAD_LIMIT) {
		if (sscanf(t, "%lu", &id[n]) == 1)
		    ++n;
	    }
	}
	fclose(f);
    }
    /* #2 get bodies */
    if (delaybody || had_bodies) {
	syslog(LOG_INFO, "%s: marked bodies %d", group->name, n);
	if (verbose > 1)
	    printf("%s: marked bodies %d\n", group->name, n);
    }
    for (i = 0; i < n; ++i)
	if (getbody(current_server, group, id[i]))
	    id[i] = 0;

    /* #3 write back ids of all articles which could not be retrieved */
    if (had_bodies) {
	if (!(f = fopen(mastr_str(filename), "w")))
	    ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for writing", mastr_str(filename));
	else {
	    for (i = 0; i < n; ++i)
		if (id[i] != 0)
		    fprintf(f, "%lu\n", id[i]);
	    fclose(f);
	}
    }
    if (delaybody || had_bodies) {
	if (verbose)
	    printf("%s: Done getting article bodies.\n", group->name);
    }
    mastr_delete(filename);
}

/** count number of colons in the string s_in. */
static int count_colons(const char *s_in) {
    int ngs;
    const char *t;

    ngs = 0;
    t = s_in;
    for(;;) {
	t += strcspn(t, ":");
	if (!*t) break;
	t++;
	ngs++;
	if (ngs < 0) {
	    /* overflow */
	    return INT_MAX;
	}
    }
    return ngs;
}

/** check if article (by Message ID) is present in spool
 \return - 0 for false (not present)
         - 1 for true (present) */
static int is_articlepresent(const char *mid /**< Message-ID of article to check */)
{
    struct stat st;
    const char *t;

    t = lookup(mid);
    if (lstat(t, &st) == 0) {
	if (st.st_size == 0) {
	    /* article was purged by XOVER-related code (illegal headers,
	     * corruption, ... - remove the message.id/NNN/ file so that we can
	     * try backfilling with fetchnews -x without a costly prior texpire
	     * -r run.
	     */
	    unlink(t);
	    return 0;
	}
	return 1;
    }
    return 0;
}

/*
 * get newsgroup from a server. "server" is the last article that
 * was previously read from this group on that server
 */
static unsigned long
getgroup(const struct server *current_server,
	/*@null@*/ struct newsgroup *g,
	unsigned long server)
{
#define HD_MAX 10
    static char *hd[HD_MAX];
    const char *hnames[HD_MAX] = { "Path: ", "Message-ID: ", "From: ",
	"Newsgroups: ", "Subject: ", "Date: ",
	"References: ", "Lines: ", "Xref: ", ""
    };

    /* order of headers in XOVER */
    enum enames { /* 0: art. no. */ h_sub = 1, h_fro, h_dat, h_mid,
	h_ref, h_byt, h_lin, h_xref };

    /* XOVER fields: 0 Subject, 1 From, 2 Date, 3 Message-ID, 4
     * References, 5 Bytes, 6 Lines, 7 Xref:full (optional) */

    unsigned long fetched, killed;
    unsigned long h;
    long n;
    unsigned long last;
    unsigned long window;	/* last ARTICLE n command sent */
    char *l;
    FILE *f;
    const char *c;
    struct stat st;
    long outstanding = 0, j;
    unsigned long i;
    unsigned long *stufftoget;
    int localmaxage = maxage;
    int maxagelimit = -1;
    const char *limitfrom = "";
    int expdays;

    if (!g)
	abort();

    if (g->first > g->last && g->first - g->last > 1)
	g->last = g->first - 1;

    if ((expdays = lookup_expiredays(g->name)) > 0) {
	if (localmaxage > expdays) {
	    maxagelimit = expdays;
	    limitfrom = "groupexpire";
	}
    } else {
	if (localmaxage > expiredays) {
	    maxagelimit = expiredays;
	    limitfrom = "global expire";
	}
    }

    if (*limitfrom && localmaxage > maxagelimit) {
	if (clamp_maxage) {
	    syslog(LOG_NOTICE, "clamping maxage for %s to %s %d",
		   g->name, limitfrom, maxagelimit);
	    localmaxage = maxagelimit;
	} else {
	    fprintf(stderr,
		   "warning: group %s: maxage of %d is inappropriate for your "
		   "applicable %s of %d. This can cause excessive downloads of "
		   "articles that were previously downloaded and expired. "
		   "Fix your configuration.\n", g->name, localmaxage, limitfrom,
		   maxagelimit);
	    syslog(LOG_WARNING,
		   "warning: group %s: maxage of %d is inappropriate for your "
		   "applicable %s of %d. This can cause excessive downloads of "
		   "articles that were previously downloaded and expired. "
		   "Fix your configuration.", g->name, localmaxage, limitfrom,
		   maxagelimit);
	}
    }

    if (server == 0ul)
	server = 1ul;

    /* skip */
    if (gs_match(current_server->group_pcre, g->name) != 1) {
	if (verbose > 1) {
	    printf("%s: skipped %s, not in only_groups_pcre\n",
		    current_server->name, g->name);
	}
	return server;
    }

    /* skip */
    if ((l = getenv("LN_SKIP_GROUPS"))) {
	char *x, *y = critstrdup(l, "getgroup");
	for (x = strtok(y, ","); x; x = strtok(NULL, ",")) {
	    if (wildmat(g->name, x)) {
		if (verbose) {
		    printf("%s: skipped %s, LN_SKIP_GROUPS=%s\n",
			    current_server->name, g->name, l);
		    syslog(LOG_INFO, "%s: skipped %s, LN_SKIP_GROUPS=%s",
			    current_server->name, g->name, l);
		}
		free(y);
		return server;
	    }
	}
	free(y);
    }

    xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", g->name);
    putaline();

    n = nntpreply(current_server);
    if (n == 498) {
	return 0;
    }
    l = lastreply();

    if (n == 411) {		/* group not available on server */
	if (verbose > 1)
	    printf("%s: no such group\n", g->name);
	return server;
    }

    if (sscanf(l, "%3ld %lu %lu %lu ", &n, &h, &window, &last) < 4 || n != 211)
    {
	fprintf(stderr, "Warning: %s: cannot parse server reply \"%s\"\n", g->name, l);
	syslog(LOG_WARNING, "Warning: %s: cannot parse server reply \"%s\"", g->name, l);
	return 0;
    }

    if (h == 0) {
	if (verbose > 1)
	    printf("%s: upstream group is empty\n", g->name);
	return server;
    }

    if (extraarticles) {
	if (server > extraarticles)
	    i = server - extraarticles;
	else
	    i = 1;
	if (i < window)
	    i = window;
	if (verbose > 1 && server > 1)
	    printf("%s: backing up from %lu to %lu\n", g->name, server, i);
	server = i;
    }

    if (server > last + 1) {
	syslog(LOG_INFO,
	       "%s: last seen article was %lu, server now has %lu-%lu",
	       g->name, server, window, last);
	if (server > last + 5) {
	    if (verbose)
		printf("%s: switched upstream servers? %lu > %lu\n",
		       g->name, server - 1, last);
	    server = window;	/* insane - recover thoroughly */
	} else {
	    if (verbose)
		printf("%s: rampant spam cancel? %lu > %lu\n", g->name, server - 1, last);
	    server = last - 5;	/* a little bit too much */
	}
    }

    if (initiallimit && server == 1 && last > server
	&& last - server > initiallimit) {
	if (verbose > 1)
	    printf("%s: skipping articles %lu-%lu inclusive (initial limit)\n",
		   g->name, server, last - initiallimit);
	syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (initial limit)",
	       g->name, server, last - initiallimit);
	server = last - initiallimit + 1;
    }

    if (artlimit && last > server && last - server > artlimit) {
	if (verbose > 1)
	    printf("%s: skipping articles %lu-%lu inclusive (article limit)\n",
		   g->name, server, last - artlimit - 1);
	syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (article limit)",
	       g->name, server, last - artlimit - 1);
	server = last - artlimit;
    }

    getmarked(current_server, g);

    if (window < server)
	window = server;
    if (window < 1)
	window = 1;
    server = window;

    if (server > last) {
	if (verbose > 1)
	    printf("%s: no new articles\n", g->name);
	syslog(LOG_INFO, "%s: no new articles\n", g->name);
	return server;
    }

    if (verbose > 1)
	printf("%s: considering articles %lu - %lu\n", g->name, server, last);
    syslog(LOG_INFO, "%s: considering articles %lu - %lu\n", g->name, server,
	   last);

    fetched = 0;
    killed = 0;

    stufftoget =
	(unsigned long *)malloc(sizeof(stufftoget[0]) * (last + 1 - server));
    if (!stufftoget) {
	ln_log(LNLOG_SERR, LNLOG_CGROUP, "not enough memory for XHDRs");
	return server;
    }
    memset(stufftoget, 0, sizeof(stufftoget[0]) * (last + 1 - server));

    if (!current_server->noxover) {
	int tmp;

	/* Try XOVER */
	xsnprintf(lineout, SIZE_lineout, "XOVER %lu-%lu\r\n", server, last);
	putaline();
	if (nntpreply(current_server) == 224) {
	    mastr *ol = mastr_new(1024);

	    debug--;
	    while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
		/*@dependent@*/ char *fields[HD_MAX];
		unsigned long art;
		char *q;

		mastr_cpy(ol, l);
		/* split xover */
		/*@+loopexec@*/
		for (i = 0; l && l[0] && i < HD_MAX; i++) {
		    char *y;
		    fields[i] = l;
		    if ((y = strchr(l, '\t'))) {
			y[0] = '\0';
			l = y + 1;
		    } else {
			l = NULL;
		    };
		};
		/*@=loopexec@*/
		/* short line -- log and skip */
		if (i < 7) {
		    ln_log(LNLOG_SWARNING, LNLOG_CTOP,
			    "%s: %s: Warning: got unparsable XOVER line from server, "
			    "too few fields (%lu): \"%s\"",
			    current_server->name, g->name, i, mastr_str(ol));
		    continue;
		}
		for (; i < HD_MAX; i++)
		    fields[i] = NULL;
		art = strtoul(fields[0], &q, 10);
		if (q && art >= server && art <= last) {
		    long artlines; /* invalid: -1 */
		    unsigned long artbytes; /* invalid: 0 */
		    if (fields[h_lin]) artlines = strtol(fields[h_lin], NULL, 10);
		    else artlines = -1;
		    if (fields[h_byt]) artbytes = strtoul(fields[h_byt], NULL, 10);
		    else artbytes = 0;

		    if (maxbytes && fields[h_byt]
			    && (strtoul(fields[h_byt], NULL, 10) > maxbytes)) {
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too many bytes (%lu > %lu)",
				g->name, art, fields[h_mid],
				strtoul(fields[h_byt], NULL, 10), maxbytes);
		    } else if (maxbytes && artbytes != 0
			    && artbytes > maxbytes) {
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too large (%lu > %lu)",
				g->name, art, fields[h_mid],
				artbytes, maxbytes);
		    } else if (maxlines && artlines != -1
			    && artlines > maxlines) {
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too many lines (%ld > %ld)",
				g->name, art, fields[h_mid],
				artlines, maxlines);
		    } else if (minlines && artlines != -1
			    && artlines < minlines) {
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too few lines (%ld < %ld)",
				g->name, art, fields[h_mid],
				artlines, minlines);
		    } else if (localmaxage && fields[h_dat]
			    && (age(fields[h_dat]) > localmaxage)) {
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too old (%d > %d) days",
				g->name, art, fields[h_mid],
				age(fields[h_dat]), localmaxage);
		    } else if (crosspostlimit && fields[h_xref] && (tmp = count_colons(fields[h_xref]) - 1) > crosspostlimit) {
			/* -1 to skip over the header's name, "Xref:" */
			killed++;
			ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
				"too many groups in Xref: header (%d > %ld)",
				g->name, art, fields[h_mid],
				tmp, crosspostlimit);
		    } else if (is_articlepresent(fields[h_mid])) {
			killed++;
			ln_log(LNLOG_SDEBUG, 4, "%s: killed %lu (%s), already fetched before",
				g->name, art, fields[h_mid]);
		    } else {
			stufftoget[outstanding] = art;
			outstanding++;
			if (verbose > 2)
			    printf("%s: will fetch %lu (%s)\n", g->name, art, fields[h_mid]);
		    }
		}
	    }
	    mastr_delete(ol);
	    debug = debugmode;
	    goto have_outstanding;
	}

	ln_log(LNLOG_SINFO, 2, "XOVER failed, trying XHDR");
    }

    xsnprintf(lineout, SIZE_lineout, "XHDR Message-ID %lu-%lu\r\n", server,
	      last);
    putaline();
    if (nntpreply(current_server) == 221) {
	debug--;
	while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
	    unsigned long art;
	    char *t;
	    art = strtoul(l, &t, 10);
	    if (t && isspace((unsigned char)*t)) {
		while (isspace((unsigned char)*t))
		    t++;
		if (art >= server && art <= last && stat(lookup(t), &st) != 0) {
		    stufftoget[outstanding] = art;
		    outstanding++;
		    if (verbose > 2)
			printf("%s: will fetch %lu (%s)\n", g->name, art, t);
		}
	    }
	}
	debug = debugmode;
	if (!l) {
	    free(stufftoget);
	    ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
		    "warning: %s: %s: server disconnect or timeout after XHDR",
		    current_server->name, g->name);
	    return 0;
	}
    } else {
	free(stufftoget);
	return server;
    }

    if (outstanding == 0) {
	free(stufftoget);
	return last + 1;
    }

    syslog(LOG_INFO, "%s: will fetch %ld article%s", g->name, outstanding, PLURAL(outstanding));
    if (verbose > 1)
	printf("%s: will fetch %ld article%s\n", g->name, outstanding, PLURAL(outstanding));

    if (minlines || maxlines) {
	xsnprintf(lineout, SIZE_lineout, "XHDR Lines %lu-%lu\r\n", server,
		  last);
	putaline();
	if (nntpreply(current_server) == 221) {
	    while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
		unsigned long art;
		long lines = 0;
		char *t;
		art = strtoul(l, &t, 10);
		if (t)
		    lines = strtol(t, NULL, 10);
		for (j = 0; j < outstanding; j++) {
		    if (art == stufftoget[j])
			break;
		}
		if (j < outstanding)
		    if ((minlines && lines < minlines)
			    || (maxlines && lines > maxlines)) {
		    stufftoget[j] = 0;
		    syslog(LOG_INFO, "%s: Killed article %lu: %ld line%s",
			   g->name, art, lines, PLURAL(lines));
		    if (verbose > 2)
			printf("%s: Killed article %lu: %ld line%s.\n",
			       g->name, art, lines, PLURAL(lines));
		    killed++;
		}
	    }
	    if (!l) {		/* timeout */
		free(stufftoget);
		ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
			"warning: %s: %s: server disconnect or timeout after XHDR Lines",
			current_server->name, g->name);
		return 0;
	    }
	}
    }

    if (maxbytes) {
	xsnprintf(lineout, SIZE_lineout, "XHDR Bytes %lu-%lu\r\n", server,
		  last);
	putaline();
	if (nntpreply(current_server) == 221) {
	    while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
		unsigned long art, bytes = 0;
		char *t;
		art = strtoul(l, &t, 10);
		if (t)
		    bytes = strtoul(t, NULL, 10);
		for (j = 0; j < outstanding; j++) {
		    if (art == stufftoget[j])
			break;
		}
		if (j < outstanding && (bytes > maxbytes)) {
		    stufftoget[j] = 0;
		    syslog(LOG_INFO, "%s: Killed article %lu (%lu > %lu bytes)",
			   g->name, art, bytes, maxbytes);
		    if (verbose > 2)
			printf("%s: Killed article %lu (%lu > %lu bytes).\n",
			       g->name, art, bytes, maxbytes);
		    killed++;
		}
	    }
	    if (!l) {		/* timeout */
		free(stufftoget);
		ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
			"warning: %s: %s: server disconnect or timeout after XHDR Bytes",
			current_server->name, g->name);

		return 0;
	    }
	}
    }

    if (localmaxage) {
	xsnprintf(lineout, SIZE_lineout, "XHDR Date %lu-%lu\r\n", server, last);
	putaline();
	if (nntpreply(current_server) == 221) {
	    debug--;
	    while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
		unsigned long art;
		long aage = 0;
		char *t;

		art = strtoul(l, &t, 10);
		if (t)
		    aage = age(t);
		for (j = 0; j < outstanding; j++) {
		    if (art == stufftoget[j])
			break;
		}
		if (j < outstanding && (aage > localmaxage)) {
		    stufftoget[j] = 0;
		    syslog(LOG_INFO,
			   "%s: Killed article %lu (%ld > %d = localmaxage)", g->name,
			   art, aage, localmaxage);
		    if (verbose > 2)
			printf("%s: Killed article %lu (%ld > %d = localmaxage).\n",
			       g->name, art, aage, localmaxage);
		    killed++;
		}
	    }

	    if (!l) {		/* timeout */
		free(stufftoget);
		ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
			"warning: %s: %s: server disconnect or timeout after XHDR Date",
			current_server->name, g->name);

		return 0;
	    }

	    debug = debugmode;
	}
    }

    /* now we have a list of articles in stufftoget[] */
    /* let's get the header and possibly bodies of these */
  have_outstanding:
    for (i = 0; outstanding > 0; i++) {
	int takethis = 1;
	int requested_body;
	const char *cmd;

	outstanding--;
	if (!stufftoget[i])
	    continue;

	if (stufftoget[i] < server) {
	    if (verbose > 2)
		printf("%s: skipping %lu - not available or too old\n",
		       g->name, stufftoget[i]);
	    syslog(LOG_INFO, "%s: skipping %lu - not available or too old",
		   g->name, stufftoget[i]);
	    continue;
	}

	debug = debugmode;
	requested_body = ((!filterfile || article_despite_filter) && !delaybody);
	cmd = requested_body ? "ARTICLE" : "HEAD";
	xsnprintf(lineout, SIZE_lineout, "%s %lu\r\n", cmd, stufftoget[i]);
	putaline();
	l = mgetaline(nntpin);
	/* timeout */
	if (!l)
	{
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		    "Server disconnection or timeout before article "
		    "could be retrieved #1");
	    free(stufftoget);
	    return 0;
	}
	/* check proper reply code */
	if (sscanf(l, "%3ld %lu", &n, &h) < 2 || ((n / 10) != 22)) {
	    if (verbose > 2)
		printf("%s %s %lu: reply %s (%ld more up in the air)\n",
		       g->name, cmd, stufftoget[i], l, outstanding);
	    syslog(LOG_INFO, "%s %s %lu: reply %s (%ld more up in the air)",
		   g->name, cmd, stufftoget[i], l, outstanding);
	    continue;
	}

	/* anything below this line will have to make sure that data is
	 * drained properly in case */

	debug--;
	if (verbose > 2)
	    printf("%s: receiving article %lu (%ld more up in the air)\n",
		   g->name, stufftoget[i], outstanding);

	for (h = 0; h < 10; h++) {
	    if (hd[h])
		free(hd[h]);
	    hd[h] = critstrdup("", "getgroup");
	}
	c = NULL;
	n = 9;			/* "other" header */
	while ((l = getfoldedline(nntpin, mgetaline)) && *l && strcmp(l, ".")) {
	    /* regexp pattern matching */
	    if (filterfile && dofilter(l)) {
		killed++;
		if (verbose > 2)
		    printf(".filtered article %lu: match on \"%s\"\n",
			    stufftoget[i], l);
		syslog(LOG_INFO, "filtered article %lu: match on \"%s\"",
		       stufftoget[i], l);
		takethis = 0;
		free(l);
		l = NULL;
		continue;
	    }

	    n = 0;
	    while (strncasecmp(l, hnames[n], strlen(hnames[n])))
		n++;
	    if (n < 9 && hd[n] && *(hd[n]))
		/* second occurance of the same recognized header
		 * is treated as if it was not listed
		 * in hnames (as "other header") */
		n = 9;
	    hd[n] = critrealloc(hd[n], strlen(hd[n]) + strlen(l) + 2,
				"Fetching article header");
	    if (strlen(hd[n]))
		strcat(hd[n], "\n");	/* RATS: ignore */
	    strcat(hd[n], l);	/* RATS: ignore */
	    if (debugmode > 1 && verbose > 3 && hnames[n] && *hnames[n])
		printf("...saw header %s\n", hnames[n]);
	    free(l);
	    l = NULL;
	} /* end while */

	/* if server ended the fetch prematurely, assume we didn't
	 * request a body to ignore - ignoring would cause timeout
	 * waiting for data that is never sent.
	 *
	 * SourceForge bug 873149, reported 2004-01-08 by Toni Viemer�,
	 * sourceforge user "skithund" */
	if (l == NULL) {
	    /* timeout - don't flush body */
	    requested_body = FALSE;
	} else if (strcmp(l, ".") == 0 && requested_body) {
	    ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "%s: %s:%lu: article without blank line after header, format violation",
		    current_server->name, g->name, stufftoget[i]);
	    requested_body = FALSE;
	}

	if (l)
	    free(l);

	if (!takethis) {
	    if (requested_body) ignore_answer(nntpin);
	    continue;		/* filtered article */
	}

	debug = debugmode;
	if (!l) {		/* timeout */
	    free(stufftoget);
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		    "Server disconnection or timeout before article "
		    "could be retrieved #2");
	    return 0;
	}

	/* check headers */
	for (h = 0; h < 6; h++) {
	    if (!hd[h] || !*(hd[h])) {
		if (verbose)
		    printf("Discarding article %lu - no %s found\n",
			   stufftoget[i], hnames[h]);
		syslog(LOG_NOTICE,
		       "Discarding article %lu - no %s found",
		       stufftoget[i], hnames[h]);
		killed++;
		if (requested_body) ignore_answer(nntpin);
		takethis = 0;
		break;
	    }
	}

	/* mandatory header missing */
	if (!takethis)
	    continue;

	if (localmaxage && age(hd[5]) > localmaxage) {
	    if (verbose > 2)
		printf("Discarding article %lu - older than %d day%s\n",
		       stufftoget[i], localmaxage, PLURAL(localmaxage));
	    syslog(LOG_INFO, "Discarding article %lu %s - older than %d day%s",
		   stufftoget[i], hd[4], localmaxage, PLURAL(localmaxage));
	    killed++;
	    if (requested_body) ignore_answer(nntpin);
	    continue;
	}

	if (minlines || maxlines) {
	    char *t;
	    t = strchr(hd[7], ' ');
	    if (t) {
		n = strtol(t, NULL, 10);
		if (minlines && n < minlines) {
		    if (verbose > 2)
			printf("Discarding article %lu - %ld < minlines\n",
			       stufftoget[i], n);
		    syslog(LOG_INFO,
			   "Discarding article %lu %s -- %ld < minlines",
			   stufftoget[i], hd[4], n);
		    killed++;
		    if (requested_body) ignore_answer(nntpin);
		    continue;
		}
		if (maxlines && n > maxlines) {
		    if (verbose > 2)
			printf("Discarding article %lu - %ld > maxlines\n",
			       stufftoget[i], n);
		    syslog(LOG_INFO,
			   "Discarding article %lu %s -- %ld > maxlines",
			   stufftoget[i], hd[4], n);
		    killed++;
		    if (requested_body) ignore_answer(nntpin);
		    continue;
		}
	    }
	}

	if (crosspostlimit) {
	    char *t;
	    t = hd[3];
	    n = 1;		/* number of groups the article is posted to */
	    while ((t = strchr(t, ',')) != NULL) {
		t++;
		n++;
	    }
	    if (crosspostlimit < n) {
		if (verbose > 2)
		    printf("Discarding article %lu - posted to %ld groups "
			   "(max. %ld)\n", stufftoget[i], n, crosspostlimit);
		syslog(LOG_INFO,
		       "Discarding article %lu %s - posted to %ld groups "
		       "(max. %ld)", stufftoget[i], hd[4], n, crosspostlimit);
		killed++;
		if (requested_body) ignore_answer(nntpin);
		continue;
	    }
	}

	/* store articles */
	f = NULL;
	c = lookup(strchr(hd[1], '<'));	/* lookup also replaces '/' with '@' */

	if (!c) {
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE, "lookup of %s failed", hd[1]);
	    if (requested_body) ignore_answer(nntpin);
	    continue;
	}

	if (!stat(c, &st)) {
	    syslog(LOG_INFO, "article %s already stored", c);
	    if (requested_body) ignore_answer(nntpin);
	    continue;		/* for some reasons, article is already there */
	} else if (errno == ENOENT) {
	    f = fopen(c, "w");
	    if (!f) {
		ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to create article %s: %m", c);
		if (requested_body) ignore_answer(nntpin);
		continue;
	    }
	} else {
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to store article %s: %m", c);
	    if (requested_body) ignore_answer(nntpin);
	    continue;
	}

	for (h = 0; h < 10; h++)
	    if (h != 8 && hd[h] && *(hd[h]))
		fprintf(f, "%s\n", hd[h]);

	h = 0;
	/* replace tabs and other odd signs with spaces */
	while (h < 8) {
	    char *p1;
	    char *p2;
	    p1 = p2 = hd[h];
	    while (p1 && *p1) {
		if (isspace((unsigned char)*p1)) {
		    *p2 = ' ';
		    do {
			p1++;
		    } while (isspace((unsigned char)*p1));
		} else {
		    *p2 = *p1++;
		}
		p2++;
	    }
	    *p2 = '\0';
	    h++;
	}

	if (fflush(f)) {
	    (void)fclose(f);
	    (void)unlink(c);
	    if (requested_body) ignore_answer(nntpin);
	    continue;
	}

	/* generate hardlinks; this procedure also increments g->last */
	store(c, f, *hd[3] ? hd[3] + strlen(hnames[3]) : "",
	      *hd[1] ? hd[1] + strlen(hnames[1]) : "");

	if (delaybody) {
	    if (fclose(f)) {
		int e = errno;
		(void)truncate(c, 0);
		(void)unlink(c);
		if (e == ENOSPC)
		    raise(SIGINT);
	    } else {
		fetched++;
	    }
	    continue;
	}

	if (!requested_body) {
	    xsnprintf(lineout, SIZE_lineout, "BODY %lu\r\n", stufftoget[i]);
	    putaline();
	    l = mgetaline(nntpin);
	    if (!l) {		/* timeout */
		(void)fflush(f);
		(void)ftruncate(fileno(f), 0);
		(void)fclose(f);
		unlink(c);
		ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
			"warning: %s: %s: server disconnect or timeout after BODY %lu",
			current_server->name, g->name, stufftoget[i]);
		free(stufftoget);
		return 0;
	    }
	    if (sscanf(l, "%3ld", &n) != 1 || (n / 10 != 22)) {
		if (verbose > 2)
		    printf("BODY %lu: reply %s\n", stufftoget[i], l);
		syslog(LOG_NOTICE, "BODY %lu: reply %s", stufftoget[i], l);
		(void)fflush(f);
		(void)ftruncate(fileno(f), 0);
		(void)fclose(f);
		unlink(c);
		continue;
	    }
	}
	debug--;
	fputs("\n", f);		/* empty line between header and body */
	while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".")) {
	    if (*l == '.')
		l++;
	    clearerr(f);
	    fputs(l, f);
	    fputc('\n', f);
	    if (feof(f)) {
		l = NULL;
		break;
	    }
	}
	debug = debugmode;
	fetched++;
	if (fflush(f)) {
	    l = NULL;
	}
	if (fclose(f)) {
	    l = NULL;
	}
	if (l == NULL) {	/* article didn't terminate with a .: error */
	    (void)truncate(c, 0);
	    (void)unlink(c);
		ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
			"warning: %s: %s: server disconnect or timeout retrieving article %lu",
			current_server->name, g->name, stufftoget[i]);
	    free(stufftoget);
	    return 0;
	}
    }

    syslog(LOG_INFO, "%s: %lu article%s fetched (to %lu), %lu killed",
	   g->name, fetched, PLURAL(fetched), g->last, killed);
    if (verbose > 1)
	printf("%s: %lu article%s fetched, %lu killed\n",
	       g->name, fetched, PLURAL(fetched), killed);
    free(stufftoget);
    return last + 1;
}

/* return 1 == success, 0 == failure */
static int expire_interesting(void) {
    DIR *d;
    struct dirent *de;
    char s[SIZE_s+1];

    xsnprintf(s, SIZE_s, "%s/interesting.groups/", spooldir);

    d = opendir(s);
    if (d == NULL) {
	ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot open %s for reading: %m", s);
	return 0;
    }

    while ((de = readdir(d))) {
	struct stat st;

	xsnprintf(s, SIZE_s, "%s/interesting.groups/%s", spooldir, de->d_name);
	if (stat(s, &st) < 0)
	    continue;
	/* reading a newsgroup changes the ctime; if the newsgroup is
	   newly created, the mtime is changed as well */
	if (((st.st_mtime == st.st_ctime) &&
	     (now - st.st_ctime > (timeout_short * SECONDS_PER_DAY))) ||
	    (now - st.st_ctime > (timeout_long * SECONDS_PER_DAY))) {
	    if (verbose > 1)
		printf("unsubscribing from %s\n", de->d_name);
	    syslog(LOG_INFO, "unsubscribing from %s (current time: %ld): "
		   "ctime age %ld, mtime age %ld", de->d_name, (long)now,
		   (long)now - st.st_ctime, (long)now - st.st_mtime);
	    unlink(s);
	}
    }

    (void)closedir(d);
    return 1;
}

static void formatserver(char *d, size_t len, const struct server *serv, const char *suffix)
{
    if (serv->port == 0 || serv->port == 119)
	xsnprintf(d, len, "%s/leaf.node/%s%s", spooldir, serv->name, suffix);
    else
	xsnprintf(d, len, "%s/leaf.node/%s:%u%s", spooldir, serv->name,
		serv->port, suffix);
}

/** get active file from current_server.
 * \returns 0 for success, non-zero for error.
 */
static int
nntpactive(struct server *current_server, time_t *stamp)
{
    struct stat st;
    char *l, *p;
    struct stringlist *groups = NULL;
    struct stringlist *helpptr = NULL;
    char timestr[64]; /* RATS: ignore */
    long reply = 0l;
    int error, merge = 0;
    char s[SIZE_s+1];
    int try_xgtitle = 1;
    static time_t cur_date;
    static int cur_date_init = 0;
    time_t t;

    if (!cur_date_init) {
	cur_date = time(NULL);
	cur_date_init = 1;
    }

    formatserver(s, SIZE_s, current_server, "");
    t = time(NULL);
    if (active && !forceactive && (stat(s, &st) == 0)) {
	if (verbose)
	    printf("%s: getting new newsgroups\n", current_server->name);
	/* to avoid a compiler warning we print out a four-digit year;
	 * but since we need only the last two digits, we skip them
	 * in the next line
	 */
	*stamp = st.st_mtime;
	(void)strftime(timestr, sizeof(timestr),
		       "%Y%m%d %H%M%S", gmtime(&st.st_mtime));
	xsnprintf(lineout, SIZE_lineout, "NEWGROUPS %s GMT\r\n", timestr + 2);
	putaline();
	/* we used to expect 231 here, but some broken servers (MC-link
	 * Custom News-server V1.06) return 215 instead.
	 * Just accept any 2XX code as success.
	 */
	if ((reply = nntpreply(current_server)) < 200 || reply >= 300) {
	    const char *e = lastreply();
	    if (!e) e = "server disconnect or timeout";
	    ln_log(LNLOG_SERR, LNLOG_CSERVER,
		    "%s: reading new newsgroups failed, reason \"%s\"",
		    current_server->name, e);
	    return -1;
	}
	while ((l = mgetaline(nntpin)) && (strcmp (l, "."))) {
	    p = l;
	    while (*p && !isspace((unsigned char)*p))
		p++;
	    if (*p)
		*p = '\0';
	    if (gs_match(current_server->group_pcre, l)) {
		merge++;
		insertgroup(l, 1, 0, cur_date);
		prependtolist(&groups, l);
	    }
	}
	if (!l) {		/* timeout */
	    ln_log(LNLOG_SERR, LNLOG_CSERVER,
		    "%s: reading new newsgroups failed, server disconnect or timeout.",
		    current_server->name);
	    return -1;
	}
	ln_log(LNLOG_SINFO, LNLOG_CSERVER,
		    "%s: got %d new newsgroups.",
		    current_server->name, merge);
	if (merge) {
	    mergegroups();	/* merge groups into active */
	}
	helpptr = groups;
	if (verbose && helpptr && current_server->descriptions)
	    printf("%s: getting newsgroup descriptions\n",
		    current_server->name);
	while (helpptr != NULL) {
	    if (current_server->descriptions) {
		error = 0;
		if (try_xgtitle) {
		    xsnprintf(lineout, SIZE_lineout, "XGTITLE %s\r\n",
			    helpptr->string);
		    putaline();
		    reply = nntpreply(current_server);
		}
		if (!try_xgtitle || reply != 282) {
		    try_xgtitle = 0;
		    xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS %s\r\n",
			      helpptr->string);
		    putaline();
		    reply = nntpreply(current_server);
		    if (reply && (reply != 215))
			error = 1;
		}
		if (!error) {
		    l = mgetaline(nntpin);
		    if (l && *l && strcmp(l, ".")) {
			p = l;
			while (*p && !isspace((unsigned char)*p))
			    p++;
			while (isspace((unsigned char)*p)) {
			    *p = '\0';
			    p++;
			}
			if (reply == 215 || reply == 282)
			    changegroupdesc(l, *p ? p : NULL);
			do {
			    l = mgetaline(nntpin);
			    error++;
			} while (l && *l && strcmp(l, "."));
			if (error > 1) {
			    current_server->descriptions = 0;
			    syslog(LOG_WARNING, "warning: %s does not process "
				   "LIST NEWSGROUPS %s correctly: use nodesc\n",
				   current_server->name, helpptr->string);
			    fprintf(stderr, "warning: %s does not process LIST "
				   "NEWSGROUPS %s correctly: use nodesc\n",
				   current_server->name, helpptr->string);
			}
		    }
		}
	    }			/* if ( current_server->descriptions ) */
	    helpptr = helpptr->next;
	}
	freelist(groups);
    } else {
	ln_log(LNLOG_SINFO, LNLOG_CSERVER,
	    "%s: getting all newsgroups (debug: active: %s, forceactive: %s)",
		current_server->name,
		active ? "set" : "nil", forceactive ? "true" : "false");
	xsnprintf(lineout, SIZE_lineout, "LIST\r\n");
	putaline();
	if (nntpreply(current_server) != 215) {
	    const char *e = lastreply();
	    if (!e) e = "server disconnect or timeout";
	    ln_log(LNLOG_SERR, LNLOG_CSERVER,
		    "%s: reading all newsgroups failed, reason \"%s\".",
		    current_server->name, e);
	    return -2;
	}
	debug--;
	while ((l = mgetaline(nntpin)) && (strcmp(l, "."))) {
	    p = l;
	    while (*p && !isspace((unsigned char)*p))
		p++;
	    while (isspace((unsigned char)*p)) {
		*p = '\0';
		p++;
	    }
	    if (gs_match(current_server->group_pcre, l)) {
		insertgroup(l, 1, 0, cur_date);
	    }
	}
	mergegroups();
	if (!l) {		/* timeout */
	    ln_log(LNLOG_SERR, LNLOG_CSERVER,
		    "%s: reading all newsgroups failed, server disconnect or timeout.",
		    current_server->name);
	    return -2;
	}
	if (current_server->descriptions) {
	    if (verbose)
		printf("%s: getting newsgroup descriptions\n",
			current_server->name);
	    xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS\r\n");
	    putaline();
	    l = mgetaline(nntpin);
	    /* correct reply starts with "215". However, INN 1.5.1 is broken
	       and immediately returns the list of groups */
	    if (l) {
		if (debug)
		    syslog(LOG_DEBUG, "<%s", l);
		reply = strtol(l, &p, 10);
		if ((reply == 215) && (*p == ' ' || *p == '\0')) {
		    l = mgetaline(nntpin);	/* get first description */
		} else if (*p != ' ' && *p != '\0') {
		    int dummy = 0;
		    /* INN 1.5.1: line already contains description */
		    (void)dummy;
		} else {
		    ln_log(LNLOG_SERR, LNLOG_CSERVER,
			    "%s: reading newsgroups descriptions failed: %s",
			    current_server->name, l);
		    ln_log(LNLOG_SERR, LNLOG_CSERVER,
			    "Workaround: Add \"nodesc = 1\" (without quotes) below the server = %s line.", current_server->name);
		    return -2;
		}
	    } else {
		ln_log(LNLOG_SERR, LNLOG_CSERVER,
			"%s: reading newsgroups descriptions failed: server disconnect or timeout.",
			current_server->name);
		return -2;
	    }
	    while (l && (strcmp(l, "."))) {
		p = l;
		while (*p && !isspace((unsigned char)*p))
		    p++;
		while (isspace((unsigned char)*p)) {
		    *p = '\0';
		    p++;
		}
		changegroupdesc(l, *p ? p : NULL);
		l = mgetaline(nntpin);
	    }
	    if (!l) {		/* timeout */
		ln_log(LNLOG_SERR, LNLOG_CSERVER,
			"%s: reading newsgroup descriptions failed, server disconnect or timeout.",
			current_server->name);
		return -2;
	    }
	}
	debug = debugmode;

	/* mark active for /this/ server fetched */
	{
	    FILE *f = fopen(s, "a");
	    if (!f) {
		ln_log(LNLOG_SERR, LNLOG_CGROUP,
			"cannot open \"%s\": %m", s);
	    } else {
		if (fclose(f))
		    ln_log(LNLOG_SERR, LNLOG_CGROUP,
			    "cannot close \"%s\": %m", s);
	    }
	}
    }
    *stamp = t;
    return 0;
}

static int
movetofailed(const char *name) {
    char s[SIZE_s + 1];

    xsnprintf(s, SIZE_s, "%s/failed.postings/%s", spooldir, name);
    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
	    "moving file %s to failed.postings", name);
    if (rename(name, s)) {
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"unable to move failed posting to %s: %m", s);
	return -1;
    } else {
	return 0;
    }
}

/*
 * post all spooled articles
 *
 * if all postings succeed, returns 1
 * if there are no postings to post, returns 1
 * if a posting is strange for some reason, returns 0
 * returns -1 if server should be skipped
 */
static int
postarticles(const struct server *current_server)
{
    struct stat st;
    char *line;
    DIR *d;
    struct dirent *de;
    FILE *f;
    int r, haveid, n;
    char *p, *q;
    int savedir;

    n = 0;

    savedir = open(".", O_RDONLY);
    if (savedir < 0) {
	ln_log(LNLOG_SERR, LNLOG_CTOP,
		"postarticles: Unable to save current working directory: %m");
	return 0;
    }

    if (chdir(spooldir) || chdir("out.going")) {
	ln_log(LNLOG_SERR, LNLOG_CTOP,
		"postarticles: Unable to cd to %s/out.going: %m",
		spooldir);
	fchdir(savedir);
	close(savedir);
	return 0;
    }

    d = opendir(".");
    if (!d) {
	ln_log(LNLOG_SERR, LNLOG_CTOP,
		"postarticles: Unable to opendir %s/out.going: %m", spooldir);
	fchdir(savedir);
	close(savedir);
	return 0;
    }

    while ((de = readdir(d)) != NULL) {
	f = NULL;
	if (!strcmp(".", de->d_name) || !strcmp("..", de->d_name)) {
	    continue;
	}
	q = NULL;
	if ((lstat(de->d_name, &st) != 0)) {
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		    "postarticles: cannot stat %s: %m",
		    de->d_name);
	    continue;
	}

	if (!S_ISREG(st.st_mode)) {
	    ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
		    "postarticles: %s is not a regular file",
		    de->d_name);
	    movetofailed(de->d_name);
	    continue;
	}

	if (!(st.st_mode & S_IRUSR)) {
	    ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
		    "postarticles: skipping %s, not complete",
		    de->d_name);
	    continue;
	}

	f = fopen(de->d_name, "r");
	if (f == NULL) {
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		    "postarticles: cannot open file %s for reading: %m",
		    de->d_name);
	    movetofailed(de->d_name);
	    continue;
	}

	p = fgetheader(f, "Newsgroups:");
	q = fgetheader(f, "Message-ID:");
	if (p == NULL || q == NULL) {
	    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		    "postarticles: file %s lacks Newsgroups "
		    "and/or Message-ID header (this cannot happen)",
		    de->d_name);
	    movetofailed(de->d_name);
	    goto free_cont;
	}

	if (!current_server->post_anygroup) {
	    char *pp = critstrdup(p, "postarticles");
	    if (!isgrouponserver(current_server, pp)) {
		ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
			"%s: postarticles: file %s: only_groups_pcre excluded "
			"or server does not carry newsgroups %s",
			current_server->name, de->d_name, p);
		free(pp);
		goto free_cont;
	    }
	    free(pp);
	}

	haveid = ismsgidonserver(current_server, q);
	switch(haveid) {
	    case 0:
		ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
			"%s: postarticles: trying to post file %s Message-ID %s",
			current_server->name, de->d_name, q);

		xsnprintf(lineout, SIZE_lineout, "POST\r\n");
		putaline();
		r = nntpreply(current_server);
		if (r != 340) {
		    char *e = lastreply();
		    if (e == NULL) {
			ln_log(LNLOG_SERR, LNLOG_CARTICLE,
				"%s: postarticles: server disconnect or timeout"
				" while trying to post file %s)",
				current_server->name, de->d_name);
			goto free_ret;
		    }

		    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
			    "%s: postarticles: server replied \"%s\" to our POST command, skipping server",
			    current_server->name, e);
		    goto free_ret;
		} else {
		    /* server replied 340, it is willing to accept our article */
		    int postok = 0;
		    debug--;
		    while ((line = getaline(f)) != NULL) {
			/* can't use putaline() here because
			   line length of lineout is restricted */
			if (line[0] == '.')
			    fputc('.', nntpout);
			fputs(line, nntpout);
			fputs("\r\n", nntpout);
		    };
		    fflush(nntpout);
		    debug = debugmode;
		    xsnprintf(lineout, SIZE_lineout, ".\r\n");
		    putaline();
		    line = mgetaline(nntpin);
		    if (!line) {	/* timeout: posting failed */
			ln_log(LNLOG_SERR, LNLOG_CARTICLE,
				"%s: postarticles: server disconnect or timeout"
				" after sending article %s)",
				current_server->name, de->d_name);
			goto free_ret;
		    }
		    line = critstrdup(line, "postarticles");
		    if (strncmp(line, "240", 3) == 0) {
			postok = 1;
		    } else if (ismsgidonserver(current_server, q) == 1) {
			syslog(LOG_NOTICE,
				"%s: postarticles: posting resulted in \"%s\", "
				"but article is available upstream, assuming OK.",
				current_server->name, line);
			printf("%s: postarticles: posting resulted in \"%s\",\n"
				"but article is available upstream, assuming OK.",
				current_server->name, line);
			postok = 1;
		    }
		    if (postok) {
			if (verbose > 2)
			    printf("%s: postarticles: POST of article %s OK\n",
				    current_server->name, de->d_name);
			n++;
			if (unlink(de->d_name)) {
			    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
				    "postarticles: unable to unlink posted article %s: %m",
				    de->d_name);
			}
		    } else {
			ln_log(LNLOG_SERR, LNLOG_CARTICLE,
				"%s: postarticles: Article file %s Message-ID %s"
				" was rejected: \"%s\"",
				current_server->name, de->d_name, q, line);
			movetofailed(de->d_name);
		    }
		    free(line);
		}
		break;
	    case 1:
		syslog(LOG_INFO, "%s: postarticles: Message-ID of %s already in use"
			" upstream -- article discarded\n",
			current_server->name, de->d_name);
		if (verbose > 2)
		    printf("%s: postarticles: %s already available upstream\n",
			    current_server->name, de->d_name);
		unlink(de->d_name);
		break;
	    case -1:
		ln_log(LNLOG_SERR, LNLOG_CARTICLE,
			"%s: postarticles: skipping server",
			current_server->name);
		goto free_ret;
	}

free_cont:
    fclose(f);
	if (p) free(p);
	if (q) free(q);
	continue;
free_ret:
	if (p) free(p);
	if (q) free(q);
	fchdir(savedir);
	close(savedir);
	closedir(d);
	return -1;
    } /* while de = readdir */
    closedir(d);
    if (verbose)
	printf("%s: %d article%s posted.\n", current_server->name, n, PLURAL(n));
    syslog(LOG_INFO, "%s: %d article%s posted.", current_server->name, n, PLURAL(n));
    fchdir(savedir);
    close(savedir);
    return 1;
}

static int
processupstream(const struct server *serv, time_t stamp)
{
    FILE *f;
    DIR *d;
    struct dirent *de;
    struct newsgroup *g;
    int havefile;
    unsigned long newserver = 0;
    char *l;
    char *oldfile;
    struct stat st1, st2;
    int have_st1 = 0;
    char s[SIZE_s+1];
    int aborting = 0;

    struct stringlist *ngs = NULL, *a, *b = NULL;

    /* read server info */
    formatserver(s, SIZE_s, serv, "");

    oldfile = critstrdup(s, "processupstream");
    havefile = 0;
    if ((f = fopen(s, "r")) != NULL) {
	if (fstat(fileno(f), &st1)) {
	    int e = errno;
	    ln_log(LNLOG_SERR, LNLOG_CSERVER, "Cannot stat %s: %s", s, strerror(e));
	} else {
	    have_st1 = 1;
	}
	/* a sorted array or a tree would be better than a list */
	ngs = NULL;
	debug--;
	if (verbose > 1)
	    printf("%s: reading server info from %s\n", serv->name, s);
	syslog(LOG_INFO, "%s: reading server info from %s", serv->name, s);
	while (((l = getaline(f)) != NULL) && (strlen(l))) {
	    a = (struct stringlist *)critmalloc(sizeof(struct stringlist)
						+ strlen(l),
						"Reading server info");
	    strcpy(a->string, l);	/* RATS: ignore */
	    a->next = NULL;
	    if (ngs == NULL)
		ngs = a;
	    else
		b->next = a;
	    b = a;
	}
	havefile = 1;
	debug = debugmode;
	fclose(f);
    }

    xsnprintf(s, SIZE_s, "%s/interesting.groups", spooldir);
    d = opendir(s);
    if (!d) {
	ln_log(LNLOG_SERR, LNLOG_CSERVER, "opendir %s: %m", s);
	free(oldfile);
	return -1;
    }

    formatserver(s, SIZE_s, serv, "~");
    if (stat(s, &st2) == 0 && (!have_st1 || st2.st_mtime >
			       st1.st_mtime) && (f = fopen(s, "r"))) {
	int e = 0;

	/* roll in changes of a previous crash */
	syslog(LOG_INFO, "Merging in %s from previous run", s);
	if (verbose > 1)
	    printf("Merging in %s from previous run\n", s);
	while (((l = getaline(f)) != NULL)) {
	    char *p, *t = strchr(l, ' ');
	    if (!t || !*++t)
		continue;
	    (void)strtoul(t, &p, 10);
	    if (p && !*p)
		replaceinlist(&ngs, l, (size_t)(t-l));
	}
	(void)fclose(f);	/* read-only file, assume no error */
	xsnprintf(s, SIZE_s, "%s/leaf.node/%s.new", spooldir, serv->name);
	f = fopen(s, "w");
	if (!f) {
	    e = 1;
	} else {
	    a = ngs;
	    while (a && a->string && !ferror(f)) {
		(void)fputs(a->string, f);
		(void)fputc('\n', f);
		a = a->next;
	    }
	    if (fclose(f))
		e = 1;
	}
	if (e) {
	    ln_log(LNLOG_SERR, LNLOG_CSERVER, "open %s: %m", s);
	    (void)unlink(s);
	    return -1;
	}
	if (rename(s, oldfile)) {
	    ln_log(LNLOG_SERR, LNLOG_CSERVER, "rename %s -> %s: %m", s, oldfile);
	    (void)unlink(s);
	    return -1;
	}
    }

    formatserver(s, SIZE_s, serv, "~");
    (void)unlink(s);
    f = fopen(s, "w");
    if (f == NULL) {
	ln_log(LNLOG_SERR, LNLOG_CSERVER, "Could not open %s for writing: %s",
		s, strerror(errno));
    } else {
	/* make sure that at least SERVERINFO~ is complete */
	if (setvbuf(f, NULL, _IOLBF, 4096)) {
	    /* try to at least use unbuffered then */
	    setvbuf(f, NULL, _IONBF, 0);
	}
    }
    while ((de = readdir(d))) {
	if (isalnum((unsigned char)*(de->d_name))) {
	    g = findgroup(de->d_name);
	    if (g != NULL) {
		unsigned long newhigh;
		xsnprintf(s, SIZE_s, "%s ", g->name);
		newhigh = 1ul;
		l = havefile ? findinlist(ngs, s) : NULL;
		if (l && *l) {
		    char *t;
		    l = strchr(l, ' ');
		    if (l) {
			newhigh = strtoul(l, &t, 10);
			if (t == l || *t)
			    newhigh = 1ul;
		    }
		}
		newserver = getgroup(serv, g, newhigh);
		/* run this independent of delaybody mode, because
		 * the admin may have switched delaybody off recently,
		 * and we still want users to be able to retrieve
		 * articles. */
		if (newserver) newhigh = newserver;
		if (f != NULL && newhigh > 0) {
		    fprintf(f, "%s %lu\n", g->name, newhigh);
		}
		if (!newserver) {
		    fprintf(stderr, "Warning: aborting fetch from %s due to previous condition.\n", serv->name);
		    syslog(LOG_WARNING, "Warning: aborting fetch from %s due to previous condition.", serv->name);
		    aborting = 1;
		    break;
		}
	    } else {
		if (verbose > 1)
		    printf("%s not found in groupinfo file\n", de->d_name);
		syslog(LOG_NOTICE, "%s not found in groupinfo file", de->d_name);
	    }
	} /* if isalnum */
    } /* while readdir */
    closedir(d);
    if (f != NULL) {
	int ren = 1;
	formatserver(s, SIZE_s, serv, "~");
	if (ferror(f))
	    ren = 0;
	if (fflush(f))
	    ren = 0;
	if (fclose(f))
	    ren = 0;
	if (!aborting) {
	    if (ren) {
		struct utimbuf ut;
		if (rename(s, oldfile)) {
		    ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot rename %s to %s: %m",
			    s, oldfile);
		}
		ut.modtime = ut.actime = stamp;
		if (utime(oldfile, &ut)) {
		    ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot set proper time for %s to %lu: %m",
			    s, (unsigned long)stamp);
		}
	    } else {
		ln_log(LNLOG_SERR, LNLOG_CSERVER, "write error on %s, old version of %s kept",
			s, oldfile);
	    }
	}
    }

    free(oldfile);
    freelist(ngs);
    return 0;
}

/*
 * checks whether all newsgroups have to be retrieved anew
 * returns 0 if yes, time of last update if not
 * mtime is the time when active was fetched fully
 * atime is the time when active was last updated
 */
static time_t
checkactive(void)
{
    struct stat st;
    char *s = activeread();

    if (stat(s, &st)) {
	free(s);
	return 0;
    }
    if ((now - st.st_mtime) < (timeout_active * SECONDS_PER_DAY)) {
	if (debugmode)
	    syslog(LOG_DEBUG,
		   "Last LIST done %d seconds ago: NEWGROUPS\n",
		   (int)(now - st.st_mtime));
	free(s);
	return st.st_atime;
    } else {
	if (debugmode)
	    syslog(LOG_DEBUG, "Last LIST done %d seconds ago: LIST\n",
		   (int)(now - st.st_mtime));
	free(s);
	return 0;
    }
}

static int
updateactive(void)
{
    struct stat st;
    struct utimbuf buf;
    FILE *f;
    char *s = activeread();
    int rc = 0;

    if (stat(s, &st)) {
	/* active.read probably doesn't exist */
	(void)unlink(s); /* delete it in case it's junk */
	if ((f = fopen(s, "w")) != NULL) {
	    if (fsync(fileno(f))) rc = -1;
	    if (fclose(f)) rc = -1;
	} else {
	    /* f == NULL, open error */
	    rc = -1;
	}
    } else {
	buf.actime = (now < st.st_atime) ? st.st_atime : now;
	/* now < update may happen through HW failures */
	buf.modtime = st.st_mtime;
	if (utime(s, &buf)) {
	    rc = -1;
	}
    }
    if (rc)
	ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot update or create %s: %m", s);
    free(s);
    return rc;
}

static void
error_refetch(const char *e) {
    ln_log(LNLOG_SERR, LNLOG_CTOP,
	    "ERROR: FETCHNEWS MUST REFETCH THE WHOLE ACTIVE FILE NEXT RUN.");
    ln_log(LNLOG_SERR, LNLOG_CTOP, "REASON: %s", e);
}

/** re-add non-expiring groups to active */
static void
addnonexpiring(void)
{
    struct stringlist *t, *l = get_grouplist();
    int expdays;

    for (t=l; t != NULL; t = t->next) {
	char *x = t->string;

	if ((expdays = lookup_expiredays(x)) >= 0) {
	    if (expdays == 0 || !(expdays = lookup_expire(x)))
		expdays = expire;
	} else {
	    expdays = -1;
	}

	if (expdays == -1) {
	    insertgroup(x, 0, 0, 0);
	}
    }
    freelist(l);
}

int
main(int argc, char **argv)
{
    /* the volatile keyboard avoids clobbering by siglongjmp */
    volatile time_t lastrun;
    volatile int rc = 0, skip_servers = 0;
    volatile int anypost = 0, waitchild = 0, quiet;
    volatile int wantpost = 0;
    struct server *current_server;
    volatile int need_refetch = 0;

    int option, reply;
    pid_t pid;

    verbose = quiet = 0;
    postonly = waitchild = 0;

    myopenlog("fetchnews");

    if (!initvars(argv[0]))
	exit(1);

    while ((option = getopt(argc, argv, "Pfhlnvx:qw")) != -1) {
	if (option == 'v') {
	    verbose++;
	    quiet = 0;
	} else if (option == 'h') {
	    usage();
	    exit(EXIT_SUCCESS);
	} else if (option == 'x') {
	    char *nptr, *endptr;
	    nptr = optarg;
	    endptr = NULL;
	    extraarticles = strtoul(nptr, &endptr, 0);
	    if (!nptr || !*nptr || !endptr || *endptr || !extraarticles) {
		usage();
		exit(1);
	    }
	    syslog(LOG_NOTICE, "fetchnews: run with option -x %lu", extraarticles);
	} else if (option == 'l') {
	    usesupplement = 0;	/* don't use supplementary servers */
	} else if (option == 'n') {
	    noexpire = 1;
	} else if (option == 'f') {
	    forceactive = 1;
	} else if (option == 'P') {
	    postonly = 1;
	} else if (option == 'q') {
	    verbose = 0;
	    quiet = 1;
	} else if (option == 'w') {
	    waitchild = 1;
	} else {
	    usage();
	    exit(1);
	}
    }

    /* Set line buffering to ensure that logging gets displayed promptly. */
    if (setvbuf(stdout, NULL, _IOLBF, 4096)) {
	/* Try to at least use unbuffered then */
	setvbuf(stdout, NULL, _IONBF, 0);
    }

    now = time(NULL);

    umask(2);

    if (!readconfig(0)) {
	fprintf(stderr, "Reading configuration failed, exiting "
	       "(see syslog for more information).\n");
	freeconfig();
	exit(1);
    }

    if (debugmode)
	syslog(LOG_DEBUG, "leafnode %s: verbosity level is %d, debugmode is %d",
		version, verbose, debugmode);
    if (verbose || debugmode) {
	printf("leafnode %s: verbosity level is %d, debugmode is %d\n",
		version, verbose, debugmode);
	if (verbose > 1 && noexpire) {
	    printf("Don't automatically unsubscribe unread newsgroups.\n");
	}
    }

    if (forceactive) {
	ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
		"Forced active fetch requested from command-line (option -f).");
    }

    if (try_lock(timeout_lock)) {
	ln_log(LNLOG_SERR, LNLOG_CTOP,
		"Cannot obtain lock file, aborting.");
	freeconfig();
	exit(1);
    }

    if (!postonly) {
	readactive();
	if (!active) {
	    addnonexpiring();
	    fakeactive(); /* we need proper lowwater/highwater marks
			     for the groups that exist */
	    ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
		    "Forced active fetch after trouble reading active file.");
	    forceactive |= 2;
	}

	readfilter(filterfile);
    }

    lastrun = 0;
    if (!postonly && !forceactive) {
	lastrun = checkactive();
	if (!lastrun) {
	    ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Active has not been fetched completely in previous run");
	    ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "or has never been fetched, forcing active fetch.");
	    forceactive |= 2;
	}
    }

    if (!postonly && !forceactive) {
	int staterror;
	struct stat st;
	char s[SIZE_s+1];

	xsnprintf(s, SIZE_s, "%s/leaf.node/groupinfo", spooldir);
	if ((staterror = stat(s, &st)) && errno != ENOENT) {
	    /* this should happen only when the problem occurs after
	     * readactive() above, but needs to be checked nonetheless */
	    ln_log(LNLOG_SERR, LNLOG_CTOP, "Cannot open %s: %m", s);
	    unlink(lockfile);
	    freeconfig();
	    exit(EXIT_FAILURE);
	}
	if (staterror || st.st_size < 7) {
	    ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
		    "Groupinfo file %s not present or too small, "
		    "forcing active fetch.", s);
	    forceactive |= 2;
	}
    }

    if (forceactive) {
	oldactive = active;
	oldactivesize = activesize;
	active = NULL;
	activesize = 0;
	if (killactiveread()) exit(1);
	addnonexpiring();
    }

    if (mysigact(SIGINT, 0, sig_int, SIGALRM) != 0)
	fprintf(stderr, "Can't catch SIGINT.\n");
    if (mysigact(SIGTERM, 0, sig_int, SIGALRM) != 0)
	fprintf(stderr, "Can't catch SIGTERM.\n");
    if (mysigact(SIGPIPE, 0, sig_int, SIGALRM) != 0)
	fprintf(stderr, "Can't catch SIGPIPE.\n");
    {
	int sig = sigsetjmp(jmpbuffer, 1);

	if (sig != 0) {
	    ln_log(LNLOG_SWARNING, LNLOG_CTOP,
		    "fetchnews: caught signal %d, shutting down.", sig);
	    nntpquit();
	    if (!rc)
		rc = 2;
	    if (forceactive) {
		error_refetch("caught signal that caused a premature abort.");
		need_refetch = 1;
	    }
	    skip_servers = 1;	/* in this case, jump the while ... loop */
	} else {
	    canjump = 1;
	}
    }

    /* remove groups that haven't been read in a long time */
    if (!noexpire)
	expire_interesting();

    mgetaline_settimeout(timeout_fetchnews);

    /* main server loop */
    for (current_server = servers;
		    !skip_servers && current_server;
		    current_server = current_server->next) {
	if (verbose) {
	    if (current_server->port)
		printf("%s: connecting to port %u...\n",
		       current_server->name, current_server->port);
	    else
		printf("%s: connecting to port nntp...\n",
		       current_server->name);
	}
	fflush(stdout);
	reply = nntpconnect(current_server);
	if (reply) {
	    int r2;
	    if (verbose) {
		int namlen = strlen(current_server->name);
		printf("%s: connected.\n", current_server->name);
		if (stat_is_evil)
		    printf("%s: server software does not implement\n"
			   "%*s  STAT <message-ID> properly,\n"
			   "%*s  using workaround with HEAD instead,\n"
			   "%*s  at the expense of bandwidth.\n",
			   current_server->name, namlen, "", namlen, "", namlen, "");
		else
		    printf("%s: using STAT <message-ID> command.\n",
			    current_server->name);
	    }

	    if (current_server->username)
		if (!authenticate(current_server) && current_server->password)
		    ln_log(LNLOG_SERR, LNLOG_CTOP,
			    "error: may have been caused by premature authentication and be rather harmless.");

	    /* Get INN's nnrpd on the phone */
	    xsnprintf(lineout, SIZE_lineout, "MODE READER\r\n");
	    putaline();
	    r2 = nntpreply(current_server);
	    if (r2 < 400) reply = r2;

	    if (reply == 498) {
		ln_log(LNLOG_SERR, LNLOG_CTOP, "%s: protocol error after sending mode reader",
		       current_server->name);
	    } else {
		if (current_server->nopost == 0) wantpost = 1;
		if (reply == 200 && current_server->nopost == 0) {
		    anypost = 1;
		    if (-1 == postarticles(current_server))
		    {
			nntpquit();
			goto connfail;
		    }
		} else if (verbose) {
		    printf("Not posting to %s: ", current_server->name);
		    if (reply != 200)
			printf("non-permission ");
		    if (current_server->nopost)
			printf("nopost-set ");
		    printf("\n");
		}
		if (!date_is_evil && !getenv("LN_SUPPRESS_DATE"))
		    check_date(current_server);
		if (!postonly && !current_server->noread) {
		    time_t stamp = lastrun;

		    /* get list of newsgroups or new newsgroups */
		    if (forceactive & 1 || current_server->updateactive) {
			int r;
			if ((r = nntpactive(current_server, &stamp))) {
			    if (forceactive & 2 || r == -2) {
				error_refetch("obtaining the active file failed.");
				need_refetch = 1;
			    }
			    rc = 1;
			}
		    } else {
			if (verbose)
			    printf("%s: not attempting to update newsgroups list\n",
				   current_server->name);
		    }

		    processupstream(current_server, stamp);
		}
	    }
	    nntpquit();
	    if (verbose)
		printf("%s: conversation completed, disconnected.\n", current_server->name);
	} else { /* reply = nntpconnect */
	    if (verbose)
		printf("%s: connection failed.\n", current_server->name);
connfail:
	    if (forceactive & 1 && current_server->updateactive && !postonly && !current_server->noread) {
		error_refetch("needed to fetch the active list from the server but couldn't connect.");
		need_refetch = 1;
	    }
	    rc = 2;
	}
	if (!usesupplement)
	    break;
    }
    mergegroups(); /* just in case we were interrupted while downloading
		      the list. */

    (void)mysigact(SIGINT, 0, SIG_IGN, 0);	/* do not siglongjmp any more */
    (void)mysigact(SIGTERM, 0, SIG_IGN, 0);	/* do not siglongjmp any more */
    (void)mysigact(SIGPIPE, 0, SIG_IGN, 0);	/* SIGPIPE should not happen below */

    if (rc != 0 && oldactive) {
	/* restore old active data to keep low/high marks */
	unsigned long i;
	for (i = 0; i < oldactivesize; i++) {
	    insertgroup(oldactive[i].name, 0, 0, 0);
	}
	mergegroups();
	killactiveread();
    }

    freeactive(oldactive);
    oldactive = NULL;
    oldactivesize = 0;

    if (anypost == 0 && wantpost != 0 && skip_servers == 0 && rc != 2) {
	const char *e = "WARNING: found no server with posting permission!";
	if (!quiet)
	    fprintf(stderr, "%s\n", e);
	syslog(LOG_WARNING, "%s", e);
    }

    if (rc == 2) {
	const char *e = "WARNING: some servers have not been queried!";
	if (!quiet)
	    fprintf(stderr, "%s\n", e);
	syslog(LOG_WARNING, "%s", e);
    }

    if (fflush(stdout)) {
	/* to avoid double logging of stuff */
	ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot flush standard output: %m");
    }

    {
	/* OK, we do a pipe trick to hold the child until the parent
	 * handed over the lock file (this must happen so it does not
	 * inadvertently get removed as stale). The child reads from the
	 * pipe and is blocked until the parent has written to the pipe.
	 * The parent writes to the pipe after it has handed over the
	 * lock.
	 */
	int pfd[2], pipeok;

	pipeok = pipe(pfd);

	if (!postonly) {
	    /* only update active.read when have read active from all servers */
	    if (writeactive()) {
		ln_log(LNLOG_SERR, LNLOG_CTOP, "Error writing groupinfo.");
		error_refetch("cannot write groupinfo file.");
		rc = 1;
		/* mark for refetch */
		killactiveread();
	    } else {
		/* active written successfully */
		if (updateactive()) {
		    error_refetch("cannot update active.read file.");
		    need_refetch = 1;
		    rc = 1;
		}

		if (need_refetch) {
		    /* mark for refetch */
		    killactiveread();
		}
	    }

#ifdef HAVE_WORKING_FORK
	    pid = waitchild ? -1 : fork();
#else
	    pid = -1; waitchild = 1;
#endif

	    switch (pid) {
		case -1:
		    if (!waitchild)
			syslog(LOG_NOTICE, "fork: %m, running on parent schedule.");
		    if (verbose)
			printf("updating overview data in the foreground...\n");
		    fixxover();
		    unlink(lockfile);
		    if (verbose)
			printf("done.\n");
		    break;

		case 0:
		    (void)setsid();
		    if (debugmode)
			syslog(LOG_DEBUG, "Process forked.");
		    if (pipeok == 0) {
			/* wait for parent to hand over the lock */
			char buf[4];
			(void)close(pfd[1]);
			/* we don't REALLY check for errors here, the worst thing
			 * that could happen (in case of a kernal bug...) to
			 * us is that we start early and delete the parent's
			 * lock file before the parent handed it over -- but
			 * at that time, we'll then exit and everything is
			 * in order, with just one bogus message in the log.
			 */
			while (read(pfd[0], buf, sizeof(buf)) < 0) {
			    if (errno != EAGAIN && errno != EINTR)
				break;
			}
			(void)close(pfd[0]); }

			fixxover();
			freeactive(active);
			if (unlink(lockfile))
			    ln_log(LNLOG_SERR, LNLOG_CTOP,
				    "unlink(\"%s\"): %m", lockfile);
			if (debugmode)
			    syslog(LOG_DEBUG, "Process done.");
			freeconfig();
			_exit(0);
			break;

		default:
			{
			    int lock_ok = handover_lock(pid);

			    if (verbose) puts("Started process to update overview data in the background.\nNetwork activity has finished.");
			    if (pipeok == 0) {
				/* tell child it has the lock */
				(void)close(pfd[0]);
				(void)writes(pfd[1], "GO");
				(void)close(pfd[1]);
			    }

			    if (lock_ok) {
				/* could not hand over lock file to child, so wait until it
				   dies */
				syslog(LOG_NOTICE, "could not hand over lockfile to child %lu: %m, "
					"waiting until child is done.",
					(unsigned long)pid);
				(void)waitpid(pid, NULL, 0);
			    } else {
				syslog(LOG_INFO, "child has process ID %lu",
					(unsigned long)pid);
			    }
			    break;
			}
	    }
	} else { /* if (postonly) */
	    unlink(lockfile);
	}
    }

    freeactive(active);
    freeconfig();
    exit(rc);
}