omi/omi/dir.c

//
//   dir.c
//
//   Oliver Fromme  <olli@fromme.com>
//   @(#)$Id: dir.c,v 1.30 1998/12/04 09:04:37 olli Exp $
//

static const char cvsid[]
    = "@(#)$Id: dir.c,v 1.30 1998/12/04 09:04:37 olli Exp $";

#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <errno.h>
#include <time.h>

#include "omi.h"
#include "dir.h"
#include "rex.h"

#include <sys/wait.h>

//
//   Unfortunately, there doesn't seem to be a portable way to
//   convert a struct tm which contains GMT into a time_t.
//   Therefore I wrote this function.  It assumes that time_t
//   complies with POSIX semantics.  The calculation of leap
//   years is a bit simplified -- it won't work correctly in
//   the year 2100...  But I promise that I'll fix it if I'm
//   still alive then.
//

int monthdaysum[12] =
	{-1, 30, 58, 89, 119, 150, 180, 211, 242, 272, 303, 333};

time_t
gmtmktime (struct tm *stm)
{
	time_t tt;

	tt = (stm->tm_year - 70) * 365 + monthdaysum[stm->tm_mon]
	    + stm->tm_mday + (stm->tm_year - 68) / 4;
	if (stm->tm_year % 4 == 0 && stm->tm_mon <= 1)
		tt--;
	return tt * 86400 + stm->tm_hour * 3600
	    + stm->tm_min * 60 + stm->tm_sec;
}

//
//   These are the functions for parsing the remote LIST output
//   (or rather: for trying to parse ...).
//
//   Unfortunately, The FTP standard (RFC 959) doesn't specify
//   a fixed, portable format for retrieving information about
//   remote files, such as file size, type and modification
//   time.  Therefore we have to parse the LIST output of the
//   remote site (which hopefully looks like a Unix ls -l) and
//   apply heuristics -- in other words:  We have to take a
//   good guess.
//
//   This particular parser actually only works well with Unix-
//   style ls -l format.  It will fail with other formats.  But
//   most FTP servers out there today use the Unix format, even
//   those which don't run Unix.  If you want to mirror a VMS-
//   style server or something else, you'll have to write the
//   appropriate parser yourself.
//
//   The parser works as follows:
//    - Ignore empty lines.
//    - Ignore lines matching "total *"
//    - Try to interpret a file entry as follows:
//         [-dl]????????? {hlinks} owner {group} size <date> name {-> slink}
//      where <date> is interpreted as follows:
//         Month day {hour:min{:sec}} {year}
//      Note that a single space in the above format can match
//      any number of white space, and that some fields can
//      contain white space themselves.  "{}" denotes optional
//      fields or parts.
//    - If the file entry match fails, try to match the line as
//      an introduction of a new subdirectory:
//         {subdir}:
//    - If that fails, too, ignore the line.
//

char *
prepare_line (char *text, int *len)
{
	//
	//   Zero-terminates the line that starts at "text"
	//   (unless it is already zero-terminated), and
	//   returns its length in "len".  The beginning of
	//   the following line is returned, or NULL if this
	//   is the last line.
	//   Depends on the fact that the whole text is zero-
	//   terminated.
	//

	char *cptr;

	if ((cptr = strpbrk(text, "\r\n"))) {
		*cptr = '\0';
		*len = cptr - text;
		cptr++;
		cptr += strspn(cptr, "\r\n");
		if (!*cptr)
			return NULL;
	}
	else
		*len = strlen(text);
	return cptr;
}

int
next_field (char **cptr)
{
	//
	//   Skip to the beginning of the next field.  Fields
	//   are separated by one or multiple tabs or spaces.
	//   If there are no further fields, the pointer will
	//   point to the terminating zero, and the result is
	//   1.  Otherwise the result is 0.
	//

	while (**cptr != ' ' && **cptr != '\t') {
		if (!**cptr)
			return 1;
		++*cptr;
	}
	do ++*cptr; while (**cptr == ' ' || **cptr == '\t');
	return !**cptr;
}

int
get_month (char *cptr)
{
	//
	//   Returns the month number (January == 0) of the
	//   3-character string pointed to by "cptr".
	//   Returns -1 if there is no match.
	//
	//   The matching has been a bit relaxed, so it also
	//   works with several common locales, in case someone
	//   runs her FTP server with non-standard $LANG or
	//   $LC_TIME.  These locales should work:  EN, DE, FR,
	//   ES, PT, IT, NL (incl. variations such as de_AT).
	//

	switch (*cptr++) {
	case 'J': case 'j':
		switch (*cptr++) {
		case 'u':
			switch (*cptr) {
			case 'l':   return 6;	// Jul --> July
			default:   return 5;	// Ju* --> June
			}
		default:
			return 0;		// J* --> January
		}
	case 'E': case 'e':   return 0;		// E* --> January (ES Enero)
	case 'G': case 'g':
		switch (*cptr++) {
		case 'e':   return 0;		// Ge* --> January (IT Gennaio)
		default:   return 5;		// G* --> June (IT Giugno)
		}
	case 'F': case 'f':   return 1;		// F* --> February
	case 'M': case 'm':
		switch (*cptr++) {
		case 'e':
			return 4;		// Me* --> May (NL Mei)
		case 'a':
			switch (*cptr) {
			case 'r':   return 2;	// Mar --> March
			default:   return 4;	// Ma* --> May
			}
		default:
			return 2;		// M* --> March (DE, NL)
		}
	case 'A': case 'a':
		switch (*cptr++) {
		case 'b':
		case 'p':
		case 'v':
			return 3;		// A[bpv]* --> April
		default:
			return 7;		// A* --> August
		}
	case 'L': case 'l':   return 6;		// L* --> July (IT Luglio)
	case 'S': case 's':   return 8;		// S* --> September
	case 'O': case 'o':   return 9;		// O* --> October
	case 'N': case 'n':   return 10;	// N* --> November
	case 'D': case 'd':   return 11;	// D* --> December
	}
	return -1;
}

int
get_perms (char *cptr, mode_t *perms, mode_t r, mode_t w, mode_t x)
{
	//
	//   Reads the next three characters and tries to
	//   parse them as permission flags (i.e. "rwx").
	//   Fills in the *perms variable, using the given
	//   r, w, and x masks.
	//
	//   "s" is handled like "x" (you don't want to mirror
	//   the set-id bit, believe me).  "t" (the so-called
	//   "sticky bit") is also handled like "x".  Upper-
	//   case "S" and "T" are handled like "-".  Apart from
	//   that, case is not significant.
	//
	//   Returns 1 on success, 0 on parse error.
	//

	switch (cptr[0]) {
		case 'r': case 'R':
			*perms |= r;
			/* FALLTHROUGH */
		case '-':
			break;
		default:
			return 0;
	}
	switch (cptr[1]) {
		case 'w': case 'W':
			*perms |= w;
			/* FALLTHROUGH */
		case '-':
			break;
		default:
			return 0;
	}
	switch (cptr[2]) {
		case 'x': case 'X': case 's': case 't':
			*perms |= x;
			/* FALLTHROUGH */
		case '-': case 'S': case 'T':
			return 1;
		default:
			return 0;
	}
}

int
match_filentry (char *line, omifile *of)
{
	//
	//   Try to interpret the line as a normal ls entry,
	//   parse the fields that we need and fill in the *of
	//   struct.
	//   Returns 1 on success, 0 on failure.
	//

	int i;
	char *eptr, *psize, *pmonth, *pday;
	struct tm date;

	switch (*line++) {
	    case '-':	of->mode = S_IFREG; break;
	    case 'd':	of->mode = S_IFDIR; break;
	    case 'l':	of->mode = S_IFLNK; break;
	    default:	return 0;
	}

	//
	//   Check the permission field.
	//
	//   If there are problems parsing the permissions,
	//   we set the OFLAG_BADPERM flag.
	//

	i = get_perms(line, &of->mode, S_IRUSR, S_IWUSR, S_IXUSR);
	i += get_perms(line + 3, &of->mode, S_IRGRP, S_IWGRP, S_IXGRP);
	i += get_perms(line + 6, &of->mode, S_IROTH, S_IWOTH, S_IXOTH);

	if (i != 3)
		of->flags |= OFLAG_BADPERM;

	if (*(line += 9) == '+')	// BSD ACL flag, see strmode(3)
		line++;
	if (*line != ' ' && *line != '\t')
		return 0;
	if (next_field(&line)) return 0;
	if (next_field(&line)) return 0;

	//
	//   Now "line" points to <owner>, <group>, or <size>
	//   (depending on the presence of the <hlinks> and
	//   <group> fields).
	//
	//   Try to locate the <size> <Month> <day> sequence.
	//

	psize = line;
	if (next_field(&line)) return 0;
	pmonth = line;
	if (next_field(&line)) return 0;
	pday = line;
	if (next_field(&line)) return 0;

	i = 2;	//   Skip at most two fields.
	while (*psize < '0' || *psize > '9'
	    || *pmonth < 'A' || *pmonth > 'S'
	    || *pday < '0' || *pday > '9'
	    || (date.tm_mon = get_month(pmonth)) < 0) {
		if (i-- == 0) return 0;
		psize = pmonth;
		pmonth = pday;
		pday = line;
		if (next_field(&line)) return 0;
	}

	of->size = strtoul(psize, &eptr, 10);
	if (*eptr != ' ' && *eptr != '\t') return 0;
	date.tm_mday = strtoul(pday, &eptr, 10);
	if ((*eptr != ' ' && *eptr != '\t') ||
	    date.tm_mday < 1 || date.tm_mday > 31) return 0;

	//
	//   Now "line" points to one of these:
	//      hh:mm{:ss year}
	//      year
	//   Where <year> could be 2 or 4 digits.
	//

	if (*line < '0' || *line > '9') return 0;
	date.tm_year = strtoul(line, &eptr, 10);
	if (*eptr == ':') {
		//
		//   It is hh:mm{:ss year}.
		//

		date.tm_hour = date.tm_year;
		date.tm_year = -1;
		line = ++eptr;
		if (*line < '0' || *line > '9') return 0;
		date.tm_min = strtoul(line, &eptr, 10);
		if (*eptr == ':') {
			//
			//   We have <seconds>, and
			//   probably also <year>.
			//

			of->tprec = 0;
			line = ++eptr;
			if (*line < '0' || *line > '9') return 0;
			date.tm_sec = strtoul(line, &eptr, 10);
			if (*eptr != ' ' && *eptr != '\t') return 0;
			line = eptr;
			if (next_field(&line)) return 0;
			if (*line >= '0' && *line <= '9') {
				psize = line;
				next_field (&psize);
				if (psize && *psize) {
					date.tm_year =
					    strtoul(line, &eptr, 10);
					if (*eptr == ' ' || *eptr == '\t')
						line = psize;
					else
						date.tm_year = -1;
				}
			}
		}
		else {
			//
			//   No seconds, no year.
			//

			of->tprec = 59;
			date.tm_sec = 0;
			line = eptr;
			if (next_field(&line)) return 0;
		}
		if (date.tm_year < 0)
			if (date.tm_mon > start.tm_mon)
				date.tm_year = start.tm_year + 1899;
			else
				date.tm_year = start.tm_year + 1900;
	}
	else if (*eptr == ' ' || *eptr == '\t') {
		//
		//   It is <year> only.
		//

		of->tprec = 86399;
		date.tm_hour = date.tm_min = date.tm_sec = 0;
		line = eptr;
		if (next_field(&line)) return 0;
	}
	else
		return 0;

	//
	//   Try to be a bit clever about the year if it's
	//   only 2 digits.  Also, change it to be the number
	//   of years since 1900.
	//

	if (date.tm_year < 1900)
		if (date.tm_year < 69)
			date.tm_year += 2000;
		else
			date.tm_year += 1900;
	if (date.tm_year < 1970) {
		//
		//   1.1.19970 0:00 is the earliest date that
		//   we can represent.
		//
		date.tm_year = 1970;
		date.tm_mon = date.tm_hour = date.tm_min = date.tm_sec = 0;
		date.tm_mday = 1;
	}
	date.tm_year -= 1900;

	if (cc->flags & MIRROR_LOCALTIME) {
		date.tm_isdst = -1;
		of->mtime = mktime(&date);
	}
	else
		of->mtime = gmtmktime(&date);

	//
	//   Now "line" should point to the filename.
	//

	i = strlen(line);
	switch (of->mode & S_IFMT) {
	case S_IFLNK:
		eptr = strstr(line, " -> ");
		if (!eptr)
			return 0;
		if (!(of->data = strndup(eptr + 4, i - (eptr - line) - 4)))
			out_of_memory();
		i = eptr - line;
		break;
	case S_IFDIR:
		of->size = 0;
		/* FALLTHROUGH */
	default:
		of->data = NULL;
	}
	if (!(of->name = strndup(line, i)))
		out_of_memory();
	return 1;
}

static omifile *
locate_dir_in_tree (char *name, omifile *root)
{
	unsigned int i;
	omifile *of;

	if (!name)
		return root;
	if (!*name || (*name == '.' && !name[1]))
		return locate_dir_in_tree(strtok(NULL, "/"), root);
	for (i = 0, of = (omifile *) root->data; i < root->size; i++, of++)
		if ((of->mode & S_IFMT) == S_IFDIR && !strcmp(name, of->name))
			return locate_dir_in_tree(strtok(NULL, "/"), of);
	return NULL;
}

static char *
build_path (const char *root, const char* suffix)
{
	//
	//   Creates a new path from the given arguments.
	//   It basically concatenates them, taking care of
	//   leading/trailing slashes.  The resulting path
	//   will always start with a slash, and never end
	//   with one.  Between `root' and `suffix' there
	//   will be exactly one slash (except if `suffix'
	//   is empty).
	//
	//   The returned path has to be free()ed when it is
	//   not needed anymore.
	//

	bool rootstartslash, midslash;
	int rootlen, suffixlen, pathlen;
	char *path, *cptr;

	pathlen = rootlen = strlen(root);
	pathlen += suffixlen = strlen(suffix);
	if (!(rootstartslash = *root == '/'))
		pathlen++;
	if (rootlen && root[rootlen - 1] == '/') {
		midslash = TRUE;
		if (*suffix == '/') {
			suffix++;
			suffixlen--;
			pathlen--;
		}
	}
	else
		if (!(midslash = *suffix == '/'))
			pathlen++;
	if (suffixlen && suffix[suffixlen - 1] == '/') {
		suffixlen--;
		pathlen--;
	}
	cptr = path = tmalloc(pathlen + 1);
	if (!rootstartslash)
		*cptr++ = '/';
	strncpy (cptr, root, rootlen);
	cptr += rootlen;
	if (!midslash)
		*cptr++ = '/';
	strncpy (cptr, suffix, suffixlen);
	cptr += suffixlen;
	if (cptr[-1] == '/')
		cptr--;
	*cptr = '\0';
	return path;
}

static int
match_exclude (char *dir, omifile *of)
{
	int dirlen, namelen, result;
	char *str;

	dirlen = strlen(dir);
	namelen = strlen(of->name);
	str = tmalloc(dirlen + namelen + 3);
	strcpy (str, dir);
	str[dirlen] = '/';
	strcpy (str + dirlen + 1, of->name);
	if ((of->mode & S_IFMT) == S_IFDIR)
		strcpy (str + dirlen + namelen + 1, "/");
	if (!(result = simple_list_traverse(globalconfig.exclude,
	    (jobfunc *) regex_job, str, 0))) {
		if (!(result = simple_list_traverse(ct->conf->exclude,
		    (jobfunc *) regex_job, str, 0))) {
			if (!(result = simple_list_traverse(cc->exclude,
			    (jobfunc *) regex_job, str, 0))) {
				if (cc->flags & MIRROR_EXCLUDEALL)
					result = MATCH_DEFAULTEXCLUDE;
				else
					result = MATCH_DEFAULTINCLUDE;
			}
		}
	}
	if ((cc->debug & DEBUG_REMOTE_REGEX) && result)
		fprintf (stderr, "remote %s: %s\n", matchdesc[result], str);
	free (str);
	return result;
}

//
//   clean_excludes() removes all directories from the tree
//   which are marked for exclusion and which don't contain
//   any files to be mirrored.
//

static int
clean_excludes (omifile *omidir)
{
	omifile *of;
	int i, size, subsize;

	of = (omifile *) omidir->data;
	size = omidir->size;
	for (i = 0; i < omidir->size; i++, of++) {
		if ((of->mode & S_IFMT) == S_IFDIR) {
			subsize = clean_excludes(of);
			if ((of->flags & OFLAG_EXCLUDE) && subsize)
				of->flags &= ~(unsigned int) OFLAG_EXCLUDE;
			else
				size--;
		}
	}
	return size;
}

//
//   parse_lslr() returns the number of entries in the
//   top-level directory that has been parsed.
//

int
parse_lslr (char *cptr, omifile *omidir, char *root)
{
	char *next, *path;
	int clen, endcolon, rootcutlen, match;
	omifile of, *od;

	rootcutlen = -1;
	od = omidir;
	path = build_path(root, "");
	while (cptr && *cptr) {
		next = prepare_line(cptr, &clen);
		if (!clen) {
			//   Ignore empty lines.
			cptr = next;
			continue;
		}
		endcolon = cptr[clen - 1] == ':'; // possibly a subdir
		if (!strncasecmp(cptr, "total ", 6) && !endcolon) {
			//   Ignore "total *" lines.
			cptr = next;
			continue;
		}
		if (match_filentry(cptr, &of)) {
			//
			//   This line is a normal ls entry.
			//   Add it to od->data.  Note that we
			//   have to be careful to exclude "."
			//   and "..".
			//

			if (!od || (of.name[0] == '.' && (of.name[1] == '\0'
			    || (of.name[1] == '.' && of.name[2] == '\0')))
			    || (match = match_exclude(path, &of)) ==
			    MATCH_EXCLUDE || ((of.mode & S_IFMT) != S_IFDIR
			    && match == MATCH_DEFAULTEXCLUDE)) {
		    		free (of.name);
				cptr = next;
				continue;
			}
			if (!(od->data = realloc(od->data,
			    (++od->size) * sizeof(omifile))))
				out_of_memory();
			if (match == MATCH_DEFAULTEXCLUDE
			    && (of.mode & S_IFMT) == S_IFDIR)
				of.flags = OFLAG_EXCLUDE;
			else
				of.flags = 0;
			memcpy ((omifile *) od->data + (od->size - 1),
			    &of, sizeof(omifile));
		}
		else if (endcolon && clen > 1) {
			//
			//   If this line starts another
			//   subdirectory, find the entry in
			//   our existing tree and continue
			//   there.
			//

			cptr[--clen] = '\0';	//   kill the trailing ':'
			if (cptr[clen - 1] == '/') {
				//
				//   If there's a trailing '/',
				//   then kill it, too.
				//

				cptr[--clen] = '\0';
			}
			if (rootcutlen < 0) {
				char *slash;

				if ((slash = strrchr(cptr, '/')))
					rootcutlen = (slash - cptr) + 1;
				else
					rootcutlen = 0;
			}
			free (path);
			path = build_path(root, cptr + rootcutlen);
			od = locate_dir_in_tree
			    (strtok(cptr + rootcutlen, "/"), omidir);
		}
		cptr = next;
	}
	free (path);
	clean_excludes (omidir);
	return omidir->size;
}

void
free_tree (omifile *omidir)
{
	omifile *of;
	int i;

	of = (omifile *) omidir->data;
	for (i = 0; i < omidir->size; i++, of++) {
		if (of->name)
			free (of->name);
		switch (of->mode & S_IFMT) {
		case S_IFDIR:
			free_tree (of);
			break;
		case S_IFLNK:
			if (of->data)
				free (of->data);
		}
	}
	if (omidir->data) {
		free (omidir->data);
		omidir->data = NULL;
	}
}

void
debug_tree (char *rootname, omifile *omiroot)
{
	omifile *of;
	char *n, type;
	int i;

	fprintf (stderr, "%s:\n", rootname);
	of = (omifile *) omiroot->data;
	for (i = 0; i < omiroot->size; i++, of++) {
		switch (of->mode & S_IFMT) {
			case S_IFDIR:	type = 'd'; break;
			case S_IFLNK:	type = 'l'; break;
			default:	type = '-';
		}
		fprintf (stderr, "%c%8lu %9lu %s", type,
		    (unsigned long) of->size, (unsigned long) of->mtime,
		    of->name);
		if (type == 'l')
			fprintf (stderr, " -> %s\n", (char *) of->data);
		else
			fprintf (stderr, "\n");
	}
	fprintf (stderr, "\n");
	of = (omifile *) omiroot->data;
	for (i = 0; i < omiroot->size; i++, of++)
		if ((of->mode & S_IFMT) == S_IFDIR) {
			n = tmalloc(strlen(rootname) + strlen(of->name) + 2);
			strcpy (n, rootname);
			strcat (n, "/");
			strcat (n, of->name);
			debug_tree (n, of);
			free (n);
		}
}

//--