xref: /dragonfly/bin/cpdup/cpdup.c (revision 6693db17)
1 /*-
2  * CPDUP.C
3  *
4  * CPDUP <options> source destination
5  *
6  * (c) Copyright 1997-1999 by Matthew Dillon and Dima Ruban.  Permission to
7  *     use and distribute based on the FreeBSD copyright.  Supplied as-is,
8  *     USE WITH EXTREME CAUTION.
9  *
10  * This program attempts to duplicate the source onto the destination as
11  * exactly as possible, retaining modify times, flags, perms, uid, and gid.
12  * It can duplicate devices, files (including hardlinks), softlinks,
13  * directories, and so forth.  It is recursive by default!  The duplication
14  * is inclusive of removal of files/directories on the destination that do
15  * not exist on the source.  This program supports a per-directory exception
16  * file called .cpignore, or a user-specified exception file.
17  *
18  * Safety features:
19  *
20  *	- does not cross partition boundries on source
21  *	- asks for confirmation on deletions unless -i0 is specified
22  *	- refuses to replace a destination directory with a source file
23  *	  unless -s0 is specified.
24  *	- terminates on error
25  *
26  * Copying features:
27  *
28  *	- does not copy file if mtime, flags, perms, and size match unless
29  *	  forced
30  *
31  *	- copies to temporary and renames-over the original, allowing
32  *	  you to update live systems
33  *
34  *	- copies uid, gid, mtime, perms, flags, softlinks, devices, hardlinks,
35  *	  and recurses through directories.
36  *
37  *	- accesses a per-directory exclusion file, .cpignore, containing
38  *	  standard wildcarded ( ? / * style, NOT regex) exclusions.
39  *
40  *	- tries to play permissions and flags smart in regards to overwriting
41  *	  schg files and doing related stuff.
42  *
43  *	- Can do MD5 consistancy checks
44  *
45  *	- Is able to do incremental mirroring/backups via hardlinks from
46  *	  the 'previous' version (supplied with -H path).
47  *
48  * $DragonFly: src/bin/cpdup/cpdup.c,v 1.32 2008/11/11 04:36:00 dillon Exp $
49  */
50 
51 /*-
52  * Example: cc -O cpdup.c -o cpdup -lmd
53  *
54  * ".MD5.CHECKSUMS" contains md5 checksumms for the current directory.
55  * This file is stored on the source.
56  */
57 
58 #include "cpdup.h"
59 #include "hclink.h"
60 #include "hcproto.h"
61 
62 #define HSIZE	8192
63 #define HMASK	(HSIZE-1)
64 #define HLSIZE	8192
65 #define HLMASK	(HLSIZE - 1)
66 
67 #define GETBUFSIZE	8192
68 #define GETPATHSIZE	2048
69 #define GETLINKSIZE	1024
70 #define GETIOSIZE	65536
71 
72 #ifndef _ST_FLAGS_PRESENT_
73 #define st_flags	st_mode
74 #endif
75 
76 typedef struct Node {
77     struct Node *no_Next;
78     struct Node *no_HNext;
79     int  no_Value;
80     char no_Name[4];
81 } Node;
82 
83 typedef struct List {
84     Node	li_Node;
85     Node	*li_Hash[HSIZE];
86 } List;
87 
88 struct hlink {
89     ino_t ino;
90     ino_t dino;
91     int	refs;
92     struct hlink *next;
93     struct hlink *prev;
94     nlink_t nlinked;
95     char name[0];
96 };
97 
98 typedef struct copy_info {
99 	char *spath;
100 	char *dpath;
101 	dev_t sdevNo;
102 	dev_t ddevNo;
103 } *copy_info_t;
104 
105 struct hlink *hltable[HLSIZE];
106 
107 void RemoveRecur(const char *dpath, dev_t devNo);
108 void InitList(List *list);
109 void ResetList(List *list);
110 char *IterateList(List *list, Node **nodeptr, int n);
111 int AddList(List *list, const char *name, int n);
112 static int getbool(const char *str);
113 static char *SplitRemote(char *path);
114 static int ChgrpAllowed(gid_t g);
115 static int OwnerMatch(struct stat *st1, struct stat *st2);
116 #ifdef _ST_FLAGS_PRESENT_
117 static int FlagsMatch(struct stat *st1, struct stat *st2);
118 #else
119 #define FlagsMatch(st1, st2)	1
120 #endif
121 static struct hlink *hltlookup(struct stat *);
122 static struct hlink *hltadd(struct stat *, const char *);
123 static char *checkHLPath(struct stat *st, const char *spath, const char *dpath);
124 static int validate_check(const char *spath, const char *dpath);
125 static int shash(const char *s);
126 static void hltdelete(struct hlink *);
127 static void hltsetdino(struct hlink *, ino_t);
128 int YesNo(const char *path);
129 static int xrename(const char *src, const char *dst, u_long flags);
130 static int xlink(const char *src, const char *dst, u_long flags);
131 static int xremove(struct HostConf *host, const char *path);
132 static int DoCopy(copy_info_t info, int depth);
133 static int ScanDir(List *list, struct HostConf *host, const char *path,
134 	int64_t *CountReadBytes, int n);
135 
136 int AskConfirmation = 1;
137 int SafetyOpt = 1;
138 int ForceOpt;
139 int DeviceOpt = 1;
140 int VerboseOpt;
141 int DirShowOpt;
142 int QuietOpt;
143 int NoRemoveOpt;
144 int UseMD5Opt;
145 int UseFSMIDOpt;
146 int SummaryOpt;
147 int CompressOpt;
148 int SlaveOpt;
149 int EnableDirectoryRetries;
150 int DstBaseLen;
151 int ValidateOpt;
152 int HardLinkCount;
153 int ssh_argc;
154 const char *ssh_argv[16];
155 int DstRootPrivs;
156 int GroupCount;
157 gid_t *GroupList;
158 const char *UseCpFile;
159 const char *UseHLPath;
160 const char *MD5CacheFile;
161 const char *FSMIDCacheFile;
162 
163 int64_t CountSourceBytes;
164 int64_t CountSourceItems;
165 int64_t CountCopiedItems;
166 int64_t CountSourceReadBytes;
167 int64_t CountTargetReadBytes;
168 int64_t CountWriteBytes;
169 int64_t CountRemovedItems;
170 int64_t CountLinkedItems;
171 
172 struct HostConf SrcHost;
173 struct HostConf DstHost;
174 
175 int
176 main(int ac, char **av)
177 {
178     int i;
179     int opt;
180     char *src = NULL;
181     char *dst = NULL;
182     char *ptr;
183     struct timeval start;
184     struct copy_info info;
185 
186     signal(SIGPIPE, SIG_IGN);
187 
188     gettimeofday(&start, NULL);
189     opterr = 0;
190     while ((opt = getopt(ac, av, ":CdF:fH:Ii:j:K:klM:mopqSs:uVvX:x")) != -1) {
191 	switch (opt) {
192 	/* TODO: sort the branches */
193 	case 'C':
194 	    CompressOpt = 1;
195 	    break;
196 	case 'v':
197 	    ++VerboseOpt;
198 	    break;
199 	case 'd':
200 	    DirShowOpt = 1;
201 	    break;
202 	case 'l':
203 	    setlinebuf(stdout);
204 	    setlinebuf(stderr);
205 	    break;
206 	case 'V':
207 	    ValidateOpt = 1;
208 	    break;
209 	case 'I':
210 	    SummaryOpt = 1;
211 	    break;
212 	case 'o':
213 	    NoRemoveOpt = 1;
214 	    break;
215 	case 'x':
216 	    UseCpFile = ".cpignore";
217 	    break;
218 	case 'X':
219 	    UseCpFile = optarg;
220 	    break;
221 	case 'H':
222 	    UseHLPath = optarg;
223 	    break;
224 	case 'F':
225 	    if (ssh_argc >= 16)
226 		fatal("too many -F options");
227 	    ssh_argv[ssh_argc++] = optarg;
228 	    break;
229 	case 'S':
230 	    SlaveOpt = 1;
231 	    break;
232 	case 'f':
233 	    ForceOpt = 1;
234 	    break;
235 	case 'i':
236 	    AskConfirmation = getbool(optarg);
237 	    break;
238 	case 'j':
239 	    DeviceOpt = getbool(optarg);
240 	    break;
241 	case 's':
242 	    SafetyOpt = getbool(optarg);
243 	    break;
244 	case 'q':
245 	    QuietOpt = 1;
246 	    break;
247 	case 'k':
248 	    UseFSMIDOpt = 1;
249 	    FSMIDCacheFile = ".FSMID.CHECK";
250 	    break;
251 	case 'K':
252 	    UseFSMIDOpt = 1;
253 	    FSMIDCacheFile = optarg;
254 	    break;
255 	case 'M':
256 	    UseMD5Opt = 1;
257 	    MD5CacheFile = optarg;
258 	    break;
259 	case 'm':
260 	    UseMD5Opt = 1;
261 	    MD5CacheFile = ".MD5.CHECKSUMS";
262 	    break;
263 	case 'u':
264 	    setvbuf(stdout, NULL, _IOLBF, 0);
265 	    break;
266 	case ':':
267 	    fatal("missing argument for option: -%c\n", optopt);
268 	    /* not reached */
269 	    break;
270 	case '?':
271 	    fatal("illegal option: -%c\n", optopt);
272 	    /* not reached */
273 	    break;
274 	default:
275 	    fatal(NULL);
276 	    /* not reached */
277 	    break;
278 	}
279     }
280     ac -= optind;
281     av += optind;
282     if (ac > 0)
283 	src = av[0];
284     if (ac > 1)
285 	dst = av[1];
286     if (ac > 2)
287 	fatal("too many arguments");
288 
289     /*
290      * If we are told to go into slave mode, run the HC protocol
291      */
292     if (SlaveOpt) {
293 	DstRootPrivs = (geteuid() == 0);
294 	hc_slave(0, 1);
295 	exit(0);
296     }
297 
298     /*
299      * Extract the source and/or/neither target [user@]host and
300      * make any required connections.
301      */
302     if (src && (ptr = SplitRemote(src)) != NULL) {
303 	SrcHost.host = src;
304 	src = ptr;
305 	if (UseMD5Opt) {
306 	    fprintf(stderr, "The MD5 options are not currently supported for remote sources\n");
307 	    exit(1);
308 	}
309 	if (hc_connect(&SrcHost) < 0)
310 	    exit(1);
311     }
312     if (dst && (ptr = SplitRemote(dst)) != NULL) {
313 	DstHost.host = dst;
314 	dst = ptr;
315 	if (UseFSMIDOpt) {
316 	    fprintf(stderr, "The FSMID options are not currently supported for remote targets\n");
317 	    exit(1);
318 	}
319 	if (hc_connect(&DstHost) < 0)
320 	    exit(1);
321     }
322 
323     /*
324      * dst may be NULL only if -m option is specified,
325      * which forces an update of the MD5 checksums
326      */
327     if (dst == NULL && UseMD5Opt == 0) {
328 	fatal(NULL);
329 	/* not reached */
330     }
331 
332     if (dst) {
333 	DstRootPrivs = (hc_geteuid(&DstHost) == 0);
334 	if (!DstRootPrivs)
335 	    GroupCount = hc_getgroups(&DstHost, &GroupList);
336     }
337 #if 0
338     /* XXXX DEBUG */
339     fprintf(stderr, "DstRootPrivs == %s\n", DstRootPrivs ? "true" : "false");
340     fprintf(stderr, "GroupCount == %d\n", GroupCount);
341     for (i = 0; i < GroupCount; i++)
342 	fprintf(stderr, "Group[%d] == %d\n", i, GroupList[i]);
343 #endif
344 
345     bzero(&info, sizeof(info));
346     if (dst) {
347 	DstBaseLen = strlen(dst);
348 	info.spath = src;
349 	info.dpath = dst;
350 	info.sdevNo = (dev_t)-1;
351 	info.ddevNo = (dev_t)-1;
352 	i = DoCopy(&info, -1);
353     } else {
354 	info.spath = src;
355 	info.dpath = NULL;
356 	info.sdevNo = (dev_t)-1;
357 	info.ddevNo = (dev_t)-1;
358 	i = DoCopy(&info, -1);
359     }
360 #ifndef NOMD5
361     md5_flush();
362 #endif
363     fsmid_flush();
364 
365     if (SummaryOpt && i == 0) {
366 	double duration;
367 	struct timeval end;
368 
369 	gettimeofday(&end, NULL);
370 #if 0
371 	/* don't count stat's in our byte statistics */
372 	CountSourceBytes += sizeof(struct stat) * CountSourceItems;
373 	CountSourceReadBytes += sizeof(struct stat) * CountSourceItems;
374 	CountWriteBytes +=  sizeof(struct stat) * CountCopiedItems;
375 	CountWriteBytes +=  sizeof(struct stat) * CountRemovedItems;
376 #endif
377 
378 	duration = (end.tv_sec - start.tv_sec);
379 	duration += (double)(end.tv_usec - start.tv_usec) / 1000000.0;
380 	if (duration == 0.0)
381 		duration = 1.0;
382 	logstd("cpdup completed successfully\n");
383 	logstd("%lld bytes source, %lld src bytes read, %lld tgt bytes read\n"
384 	       "%lld bytes written (%.1fX speedup)\n",
385 	    (long long)CountSourceBytes,
386 	    (long long)CountSourceReadBytes,
387 	    (long long)CountTargetReadBytes,
388 	    (long long)CountWriteBytes,
389 	    ((double)CountSourceBytes * 2.0) / ((double)(CountSourceReadBytes + CountTargetReadBytes + CountWriteBytes)));
390  	logstd("%lld source items, %lld items copied, %lld items linked, "
391 	       "%lld things deleted\n",
392 	    (long long)CountSourceItems,
393 	    (long long)CountCopiedItems,
394 	    (long long)CountLinkedItems,
395 	    (long long)CountRemovedItems);
396 	logstd("%.1f seconds %5d Kbytes/sec synced %5d Kbytes/sec scanned\n",
397 	    duration,
398 	    (int)((CountSourceReadBytes + CountTargetReadBytes + CountWriteBytes) / duration  / 1024.0),
399 	    (int)(CountSourceBytes / duration / 1024.0));
400     }
401     exit((i == 0) ? 0 : 1);
402 }
403 
404 static int
405 getbool(const char *str)
406 {
407     if (strcmp(str, "0") == 0)
408 	return (0);
409     if (strcmp(str, "1") == 0)
410 	return (1);
411     fatal("option requires boolean argument (0 or 1): -%c\n", optopt);
412     /* not reached */
413     return (0);
414 }
415 
416 /*
417  * Check if path specifies a remote path, using the same syntax as scp(1),
418  * i.e. a path is considered remote if the first colon is not preceded by
419  * a slash, so e.g. "./foo:bar" is considered local.
420  * If a remote path is detected, the colon is replaced with a null byte,
421  * and the return value is a pointer to the next character.
422  * Otherwise NULL is returned.
423  */
424 static char *
425 SplitRemote(char *path)
426 {
427     int cindex;
428 
429     if (path[(cindex = strcspn(path, ":/"))] == ':') {
430 	path[cindex++] = 0;
431 	return (path + cindex);
432     }
433     return (NULL);
434 }
435 
436 /*
437  * Check if group g is in our GroupList.
438  *
439  * Typically the number of groups a user belongs to isn't large
440  * enough to warrant more effort than a simple linear search.
441  * However, we perform an optimization by moving a group to the
442  * top of the list when we have a hit.  This assumes that there
443  * isn't much variance in the gids of files that a non-root user
444  * copies.  So most of the time the search will terminate on the
445  * first element of the list.
446  */
447 static int
448 ChgrpAllowed(gid_t g)
449 {
450     int i;
451 
452     for (i = 0; i < GroupCount; i++)
453 	if (GroupList[i] == g) {
454 	    if (i > 0) {
455 		/* Optimize: Move g to the front of the list. */
456 		for (; i > 0; i--)
457 		    GroupList[i] = GroupList[i - 1];
458 		GroupList[0] = g;
459 	    }
460 	    return (1);
461 	}
462     return (0);
463 }
464 
465 /*
466  * The following two functions return true if the ownership (UID + GID)
467  * or the flags of two files match, respectively.
468  *
469  * Only perform weak checking if we don't have sufficient privileges on
470  * the target machine, so we don't waste transfers with things that are
471  * bound to fail anyway.
472  */
473 static int
474 OwnerMatch(struct stat *st1, struct stat *st2)
475 {
476     if (DstRootPrivs)
477 	/* Both UID and GID must match. */
478 	return (st1->st_uid == st2->st_uid && st1->st_gid == st2->st_gid);
479     else
480 	/* Ignore UID, and also ignore GID if we can't chgrp to that group. */
481 	return (st1->st_gid == st2->st_gid || !ChgrpAllowed(st1->st_gid));
482 }
483 
484 #ifdef _ST_FLAGS_PRESENT_
485 static int
486 FlagsMatch(struct stat *st1, struct stat *st2)
487 {
488     if (DstRootPrivs)
489 	return (st1->st_flags == st2->st_flags);
490     else
491 	/* Only consider the user-settable flags. */
492 	return (((st1->st_flags ^ st2->st_flags) & UF_SETTABLE) == 0);
493 }
494 #endif
495 
496 
497 static struct hlink *
498 hltlookup(struct stat *stp)
499 {
500     struct hlink *hl;
501     int n;
502 
503     n = stp->st_ino & HLMASK;
504 
505     for (hl = hltable[n]; hl; hl = hl->next) {
506         if (hl->ino == stp->st_ino) {
507 	    ++hl->refs;
508 	    return hl;
509 	}
510     }
511 
512     return NULL;
513 }
514 
515 static struct hlink *
516 hltadd(struct stat *stp, const char *path)
517 {
518     struct hlink *new;
519     int plen = strlen(path);
520     int n;
521 
522     new = malloc(offsetof(struct hlink, name[plen + 1]));
523     if (new == NULL) {
524         fprintf(stderr, "out of memory\n");
525         exit(EXIT_FAILURE);
526     }
527     ++HardLinkCount;
528 
529     /* initialize and link the new element into the table */
530     new->ino = stp->st_ino;
531     new->dino = (ino_t)-1;
532     new->refs = 1;
533     bcopy(path, new->name, plen + 1);
534     new->nlinked = 1;
535     new->prev = NULL;
536     n = stp->st_ino & HLMASK;
537     new->next = hltable[n];
538     if (hltable[n])
539         hltable[n]->prev = new;
540     hltable[n] = new;
541 
542     return new;
543 }
544 
545 static void
546 hltsetdino(struct hlink *hl, ino_t inum)
547 {
548     hl->dino = inum;
549 }
550 
551 static void
552 hltdelete(struct hlink *hl)
553 {
554     assert(hl->refs == 1);
555     --hl->refs;
556     if (hl->prev) {
557         if (hl->next)
558             hl->next->prev = hl->prev;
559         hl->prev->next = hl->next;
560     } else {
561         if (hl->next)
562             hl->next->prev = NULL;
563 
564         hltable[hl->ino & HLMASK] = hl->next;
565     }
566     --HardLinkCount;
567     free(hl);
568 }
569 
570 static void
571 hltrels(struct hlink *hl)
572 {
573     assert(hl->refs == 1);
574     --hl->refs;
575 }
576 
577 /*
578  * If UseHLPath is defined check to see if the file in question is
579  * the same as the source file, and if it is return a pointer to the
580  * -H path based file for hardlinking.  Else return NULL.
581  */
582 static char *
583 checkHLPath(struct stat *st1, const char *spath, const char *dpath)
584 {
585     struct stat sthl;
586     char *hpath;
587     int error;
588 
589     asprintf(&hpath, "%s%s", UseHLPath, dpath + DstBaseLen);
590 
591     /*
592      * stat info matches ?
593      */
594     if (hc_stat(&DstHost, hpath, &sthl) < 0 ||
595 	st1->st_size != sthl.st_size ||
596 	st1->st_mtime != sthl.st_mtime ||
597 	!OwnerMatch(st1, &sthl) ||
598 	!FlagsMatch(st1, &sthl)
599     ) {
600 	free(hpath);
601 	return(NULL);
602     }
603 
604     /*
605      * If ForceOpt or ValidateOpt is set we have to compare the files
606      */
607     if (ForceOpt || ValidateOpt) {
608 	error = validate_check(spath, hpath);
609 	if (error) {
610 	    free(hpath);
611 	    hpath = NULL;
612 	}
613     }
614     return(hpath);
615 }
616 
617 /*
618  * Return 0 if the contents of the file <spath> matches the contents of
619  * the file <dpath>.
620  */
621 static int
622 validate_check(const char *spath, const char *dpath)
623 {
624     int error;
625     int fd1;
626     int fd2;
627 
628     fd1 = hc_open(&SrcHost, spath, O_RDONLY, 0);
629     fd2 = hc_open(&DstHost, dpath, O_RDONLY, 0);
630     error = -1;
631 
632     if (fd1 >= 0 && fd2 >= 0) {
633 	int n;
634 	int x;
635 	char *iobuf1 = malloc(GETIOSIZE);
636 	char *iobuf2 = malloc(GETIOSIZE);
637 
638 	while ((n = hc_read(&SrcHost, fd1, iobuf1, GETIOSIZE)) > 0) {
639 	    CountSourceReadBytes += n;
640 	    x = hc_read(&DstHost, fd2, iobuf2, GETIOSIZE);
641 	    if (x > 0)
642 		    CountTargetReadBytes += x;
643 	    if (x != n)
644 		break;
645 	    if (bcmp(iobuf1, iobuf2, n) != 0)
646 		break;
647 	}
648 	free(iobuf1);
649 	free(iobuf2);
650 	if (n == 0)
651 	    error = 0;
652     }
653     if (fd1 >= 0)
654 	hc_close(&SrcHost, fd1);
655     if (fd2 >= 0)
656 	hc_close(&DstHost, fd2);
657     return (error);
658 }
659 
660 int
661 DoCopy(copy_info_t info, int depth)
662 {
663     const char *spath = info->spath;
664     const char *dpath = info->dpath;
665     dev_t sdevNo = info->sdevNo;
666     dev_t ddevNo = info->ddevNo;
667     struct stat st1;
668     struct stat st2;
669     unsigned long st2_flags;
670     int r, mres, fres, st2Valid;
671     struct hlink *hln;
672     u_int64_t size;
673 
674     r = mres = fres = st2Valid = 0;
675     st2_flags = 0;
676     size = 0;
677     hln = NULL;
678 
679     if (hc_lstat(&SrcHost, spath, &st1) != 0) {
680 	r = 1;
681 	goto done;
682     }
683 #ifdef SF_SNAPSHOT
684     /* skip snapshot files because they're sparse and _huge_ */
685     if (st1.st_flags & SF_SNAPSHOT)
686        return(0);
687 #endif
688     st2.st_mode = 0;	/* in case lstat fails */
689     st2.st_flags = 0;	/* in case lstat fails */
690     if (dpath && hc_lstat(&DstHost, dpath, &st2) == 0) {
691 	st2Valid = 1;
692 #ifdef _ST_FLAGS_PRESENT_
693 	st2_flags = st2.st_flags;
694 #endif
695     }
696 
697     if (S_ISREG(st1.st_mode)) {
698 	size = st1.st_size;
699     }
700 
701     /*
702      * Handle hardlinks
703      */
704 
705     if (S_ISREG(st1.st_mode) && st1.st_nlink > 1 && dpath) {
706         if ((hln = hltlookup(&st1)) != NULL) {
707             hln->nlinked++;
708 
709             if (st2Valid) {
710                 if (st2.st_ino == hln->dino) {
711 		    /*
712 		     * hard link is already correct, nothing to do
713 		     */
714 		    if (VerboseOpt >= 3)
715 			logstd("%-32s nochange\n", (dpath) ? dpath : spath);
716                     if (hln->nlinked == st1.st_nlink) {
717                         hltdelete(hln);
718 			hln = NULL;
719 		    }
720 		    CountSourceItems++;
721 		    r = 0;
722 		    goto done;
723                 } else {
724 		    /*
725 		     * hard link is not correct, attempt to unlink it
726 		     */
727                     if (xremove(&DstHost, dpath) < 0) {
728 			logerr("%-32s hardlink: unable to unlink: %s\n",
729 			    ((dpath) ? dpath : spath), strerror(errno));
730                         hltdelete(hln);
731 			hln = NULL;
732 			++r;
733 			goto done;
734 		    }
735                 }
736             }
737 
738             if (xlink(hln->name, dpath, st1.st_flags) < 0) {
739 		int tryrelink = (errno == EMLINK);
740 		logerr("%-32s hardlink: unable to link to %s: %s\n",
741 		    (dpath ? dpath : spath), hln->name, strerror(errno)
742 		);
743                 hltdelete(hln);
744                 hln = NULL;
745 		if (tryrelink) {
746 		    logerr("%-20s hardlink: will attempt to copy normally\n");
747 		    goto relink;
748 		}
749 		++r;
750             } else {
751                 if (hln->nlinked == st1.st_nlink) {
752                     hltdelete(hln);
753 		    hln = NULL;
754 		}
755                 if (r == 0) {
756 		    if (VerboseOpt) {
757 			logstd("%-32s hardlink: %s\n",
758 			    (dpath ? dpath : spath),
759 			    (st2Valid ? "relinked" : "linked")
760 			);
761 		    }
762 		    CountSourceItems++;
763 		    CountCopiedItems++;
764 		    r = 0;
765 		    goto done;
766 		}
767             }
768         } else {
769 	    /*
770 	     * first instance of hardlink must be copied normally
771 	     */
772 relink:
773             hln = hltadd(&st1, dpath);
774 	}
775     }
776 
777     /*
778      * Do we need to copy the file/dir/link/whatever?  Early termination
779      * if we do not.  Always redo links.  Directories are always traversed
780      * except when the FSMID options are used.
781      *
782      * NOTE: st2Valid is true only if dpath != NULL *and* dpath stats good.
783      */
784 
785     if (
786 	st2Valid
787 	&& st1.st_mode == st2.st_mode
788 	&& FlagsMatch(&st1, &st2)
789     ) {
790 	if (S_ISLNK(st1.st_mode) || S_ISDIR(st1.st_mode)) {
791 	    /*
792 	     * If FSMID tracking is turned on we can avoid recursing through
793 	     * an entire directory subtree if the FSMID matches.
794 	     */
795 #ifdef _ST_FSMID_PRESENT_
796 	    if (ForceOpt == 0 &&
797 		(UseFSMIDOpt && (fres = fsmid_check(st1.st_fsmid, dpath)) == 0)
798 	    ) {
799 		if (VerboseOpt >= 3) {
800 		    if (UseFSMIDOpt) /* always true!?! */
801 			logstd("%-32s fsmid-nochange\n", (dpath ? dpath : spath));
802 		    else
803 			logstd("%-32s nochange\n", (dpath ? dpath : spath));
804 		}
805 		r = 0;
806 		goto done;
807 	    }
808 #endif
809 	} else {
810 	    if (ForceOpt == 0 &&
811 		st1.st_size == st2.st_size &&
812 		st1.st_mtime == st2.st_mtime &&
813 		OwnerMatch(&st1, &st2)
814 #ifndef NOMD5
815 		&& (UseMD5Opt == 0 || !S_ISREG(st1.st_mode) ||
816 		    (mres = md5_check(spath, dpath)) == 0)
817 #endif
818 #ifdef _ST_FSMID_PRESENT_
819 		&& (UseFSMIDOpt == 0 ||
820 		    (fres = fsmid_check(st1.st_fsmid, dpath)) == 0)
821 #endif
822 		&& (ValidateOpt == 0 || !S_ISREG(st1.st_mode) ||
823 		    validate_check(spath, dpath) == 0)
824 	    ) {
825 		/*
826 		 * The files are identical, but if we are running as
827 		 * root we might need to adjust ownership/group/flags.
828 		 */
829 		int changedown = 0;
830 		int changedflags = 0;
831 
832                 if (hln)
833 		    hltsetdino(hln, st2.st_ino);
834 
835 		if (!OwnerMatch(&st1, &st2)) {
836 		    hc_chown(&DstHost, dpath, st1.st_uid, st1.st_gid);
837 		    changedown = 1;
838 		}
839 #ifdef _ST_FLAGS_PRESENT_
840 		if (!FlagsMatch(&st1, &st2)) {
841 		    hc_chflags(&DstHost, dpath, st1.st_flags);
842 		    changedflags = 1;
843 		}
844 #endif
845 		if (VerboseOpt >= 3) {
846 #ifndef NOMD5
847 		    if (UseMD5Opt) {
848 			logstd("%-32s md5-nochange",
849 				(dpath ? dpath : spath));
850 		    } else
851 #endif
852 		    if (UseFSMIDOpt) {
853 			logstd("%-32s fsmid-nochange",
854 				(dpath ? dpath : spath));
855 		    } else if (ValidateOpt) {
856 			logstd("%-32s nochange (contents validated)",
857 				(dpath ? dpath : spath));
858 		    } else {
859 			logstd("%-32s nochange", (dpath ? dpath : spath));
860 		    }
861 		    if (changedown)
862 			logstd(" (uid/gid differ)");
863 		    if (changedflags)
864 			logstd(" (flags differ)");
865 		    logstd("\n");
866 		}
867 		CountSourceBytes += size;
868 		CountSourceItems++;
869 		r = 0;
870 		goto done;
871 	    }
872 	}
873     }
874     if (st2Valid && !S_ISDIR(st1.st_mode) && S_ISDIR(st2.st_mode)) {
875 	if (SafetyOpt) {
876 	    logerr("%-32s SAFETY - refusing to copy file over directory\n",
877 		(dpath ? dpath : spath)
878 	    );
879 	    ++r;		/* XXX */
880 	    r = 0;
881 	    goto done; 		/* continue with the cpdup anyway */
882 	}
883 	if (QuietOpt == 0 || AskConfirmation) {
884 	    logstd("%-32s WARNING: non-directory source will blow away\n"
885 		   "%-32s preexisting dest directory, continuing anyway!\n",
886 		   ((dpath) ? dpath : spath), "");
887 	}
888 	if (dpath)
889 	    RemoveRecur(dpath, ddevNo);
890 	st2Valid = 0;
891     }
892 
893     /*
894      * The various comparisons failed, copy it.
895      */
896     if (S_ISDIR(st1.st_mode)) {
897 	int skipdir = 0;
898 
899 	if (fres < 0)
900 	    logerr("%-32s/ fsmid-CHECK-FAILED\n", (dpath) ? dpath : spath);
901 
902 	if (dpath) {
903 	    if (!st2Valid || S_ISDIR(st2.st_mode) == 0) {
904 		if (st2Valid)
905 		    xremove(&DstHost, dpath);
906 		if (hc_mkdir(&DstHost, dpath, st1.st_mode | 0700) != 0) {
907 		    logerr("%s: mkdir failed: %s\n",
908 			(dpath ? dpath : spath), strerror(errno));
909 		    r = 1;
910 		    skipdir = 1;
911 		}
912 		if (hc_lstat(&DstHost, dpath, &st2) != 0) {
913 		    logerr("%s: lstat of newly made dir failed: %s\n",
914 			(dpath ? dpath : spath), strerror(errno));
915 		    st2Valid = 0;
916 		    r = 1;
917 		    skipdir = 1;
918 		}
919 		else {
920 		    st2Valid = 1;
921 		    if (!OwnerMatch(&st1, &st2) &&
922 			hc_chown(&DstHost, dpath, st1.st_uid, st1.st_gid) != 0
923 		    ) {
924 			logerr("%s: chown of newly made dir failed: %s\n",
925 			    (dpath ? dpath : spath), strerror(errno));
926 			r = 1;
927 			/* Note that we should not set skipdir = 1 here. */
928 		    }
929 		}
930 		CountCopiedItems++;
931 	    } else {
932 		/*
933 		 * Directory must be scanable by root for cpdup to
934 		 * work.  We'll fix it later if the directory isn't
935 		 * supposed to be readable ( which is why we fixup
936 		 * st2.st_mode to match what we did ).
937 		 */
938 		if ((st2.st_mode & 0700) != 0700) {
939 		    hc_chmod(&DstHost, dpath, st2.st_mode | 0700);
940 		    st2.st_mode |= 0700;
941 		}
942 		if (VerboseOpt >= 2)
943 		    logstd("%s\n", dpath ? dpath : spath);
944 	    }
945 	}
946 
947 	/*
948 	 * When copying a directory, stop if the source crosses a mount
949 	 * point.
950 	 */
951 	if (sdevNo != (dev_t)-1 && st1.st_dev != sdevNo)
952 	    skipdir = 1;
953 	else
954 	    sdevNo = st1.st_dev;
955 
956 	/*
957 	 * When copying a directory, stop if the destination crosses
958 	 * a mount point.
959 	 *
960 	 * The target directory will have been created and stat'd
961 	 * for st2 if it did not previously exist.   st2Valid is left
962 	 * as a flag.  If the stat failed st2 will still only have its
963 	 * default initialization.
964 	 *
965 	 * So we simply assume here that the directory is within the
966 	 * current target mount if we had to create it (aka st2Valid is 0)
967 	 * and we leave ddevNo alone.
968 	 */
969 	if (st2Valid) {
970 	    if (ddevNo != (dev_t)-1 && st2.st_dev != ddevNo)
971 		skipdir = 1;
972 	    else
973 		ddevNo = st2.st_dev;
974 	}
975 
976 	if (!skipdir) {
977 	    List *list = malloc(sizeof(List));
978 	    Node *node = NULL;
979 	    char *name;
980 
981 	    if (DirShowOpt)
982 		logstd("Scanning %s ...\n", spath);
983 	    InitList(list);
984 	    if (ScanDir(list, &SrcHost, spath, &CountSourceReadBytes, 0) == 0) {
985 		while ((name = IterateList(list, &node, 0)) != NULL) {
986 		    char *nspath;
987 		    char *ndpath = NULL;
988 
989 		    nspath = mprintf("%s/%s", spath, name);
990 		    if (dpath)
991 			ndpath = mprintf("%s/%s", dpath, name);
992 
993 		    info->spath = nspath;
994 		    info->dpath = ndpath;
995 		    info->sdevNo = sdevNo;
996 		    info->ddevNo = ddevNo;
997 		    if (depth < 0)
998 			r += DoCopy(info, depth);
999 		    else
1000 			r += DoCopy(info, depth + 1);
1001 		    free(nspath);
1002 		    if (ndpath)
1003 			free(ndpath);
1004 		    info->spath = NULL;
1005 		    info->dpath = NULL;
1006 		}
1007 
1008 		/*
1009 		 * Remove files/directories from destination that do not appear
1010 		 * in the source.
1011 		 */
1012 		if (dpath && ScanDir(list, &DstHost, dpath,
1013 			&CountTargetReadBytes, 3) == 0) {
1014 		    while ((name = IterateList(list, &node, 3)) != NULL) {
1015 			/*
1016 			 * If object does not exist in source or .cpignore
1017 			 * then recursively remove it.
1018 			 */
1019 			char *ndpath;
1020 
1021 			ndpath = mprintf("%s/%s", dpath, name);
1022 			RemoveRecur(ndpath, ddevNo);
1023 			free(ndpath);
1024 		    }
1025 		}
1026 	    }
1027 	    ResetList(list);
1028 	    free(list);
1029 	}
1030 
1031 	if (dpath && st2Valid) {
1032 	    struct timeval tv[2];
1033 
1034 	    if (ForceOpt || !OwnerMatch(&st1, &st2))
1035 		hc_chown(&DstHost, dpath, st1.st_uid, st1.st_gid);
1036 	    if (st1.st_mode != st2.st_mode)
1037 		hc_chmod(&DstHost, dpath, st1.st_mode);
1038 #ifdef _ST_FLAGS_PRESENT_
1039 	    if (!FlagsMatch(&st1, &st2))
1040 		hc_chflags(&DstHost, dpath, st1.st_flags);
1041 #endif
1042 	    if (ForceOpt || st1.st_mtime != st2.st_mtime) {
1043 		bzero(tv, sizeof(tv));
1044 		tv[0].tv_sec = st1.st_mtime;
1045 		tv[1].tv_sec = st1.st_mtime;
1046 		hc_utimes(&DstHost, dpath, tv);
1047 	    }
1048 	}
1049     } else if (dpath == NULL) {
1050 	/*
1051 	 * If dpath is NULL, we are just updating the MD5
1052 	 */
1053 #ifndef NOMD5
1054 	if (UseMD5Opt && S_ISREG(st1.st_mode)) {
1055 	    mres = md5_check(spath, NULL);
1056 
1057 	    if (VerboseOpt > 1) {
1058 		if (mres < 0)
1059 		    logstd("%-32s md5-update\n", (dpath) ? dpath : spath);
1060 		else
1061 		    logstd("%-32s md5-ok\n", (dpath) ? dpath : spath);
1062 	    } else if (!QuietOpt && mres < 0) {
1063 		logstd("%-32s md5-update\n", (dpath) ? dpath : spath);
1064 	    }
1065 	}
1066 #endif
1067     } else if (S_ISREG(st1.st_mode)) {
1068 	char *path;
1069 	char *hpath;
1070 	int fd1;
1071 	int fd2;
1072 
1073 	if (st2Valid)
1074 		path = mprintf("%s.tmp%d", dpath, (int)getpid());
1075 	else
1076 		path = mprintf("%s", dpath);
1077 
1078 	/*
1079 	 * Handle check failure message.
1080 	 */
1081 #ifndef NOMD5
1082 	if (mres < 0)
1083 	    logerr("%-32s md5-CHECK-FAILED\n", (dpath) ? dpath : spath);
1084 	else
1085 #endif
1086 	if (fres < 0)
1087 	    logerr("%-32s fsmid-CHECK-FAILED\n", (dpath) ? dpath : spath);
1088 
1089 	/*
1090 	 * Not quite ready to do the copy yet.  If UseHLPath is defined,
1091 	 * see if we can hardlink instead.
1092 	 *
1093 	 * If we can hardlink, and the target exists, we have to remove it
1094 	 * first or the hardlink will fail.  This can occur in a number of
1095 	 * situations but most typically when the '-f -H' combination is
1096 	 * used.
1097 	 */
1098 	if (UseHLPath && (hpath = checkHLPath(&st1, spath, dpath)) != NULL) {
1099 		if (st2Valid)
1100 			xremove(&DstHost, dpath);
1101 		if (hc_link(&DstHost, hpath, dpath) == 0) {
1102 			++CountLinkedItems;
1103 			if (VerboseOpt) {
1104 			    logstd("%-32s hardlinked(-H)\n",
1105 				   (dpath ? dpath : spath));
1106 			}
1107 			free(hpath);
1108 			goto skip_copy;
1109 		}
1110 		/*
1111 		 * Shucks, we may have hit a filesystem hard linking limit,
1112 		 * we have to copy instead.
1113 		 */
1114 		free(hpath);
1115 	}
1116 
1117 	if ((fd1 = hc_open(&SrcHost, spath, O_RDONLY, 0)) >= 0) {
1118 	    if ((fd2 = hc_open(&DstHost, path, O_WRONLY|O_CREAT|O_EXCL, 0600)) < 0) {
1119 		/*
1120 		 * There could be a .tmp file from a previously interrupted
1121 		 * run, delete and retry.  Fail if we still can't get at it.
1122 		 */
1123 #ifdef _ST_FLAGS_PRESENT_
1124 		hc_chflags(&DstHost, path, 0);
1125 #endif
1126 		hc_remove(&DstHost, path);
1127 		fd2 = hc_open(&DstHost, path, O_WRONLY|O_CREAT|O_EXCL|O_TRUNC, 0600);
1128 	    }
1129 	    if (fd2 >= 0) {
1130 		const char *op;
1131 		char *iobuf1 = malloc(GETIOSIZE);
1132 		int n;
1133 
1134 		/*
1135 		 * Matt: What about holes?
1136 		 */
1137 		op = "read";
1138 		while ((n = hc_read(&SrcHost, fd1, iobuf1, GETIOSIZE)) > 0) {
1139 		    op = "write";
1140 		    if (hc_write(&DstHost, fd2, iobuf1, n) != n)
1141 			break;
1142 		    op = "read";
1143 		}
1144 		hc_close(&DstHost, fd2);
1145 		if (n == 0) {
1146 		    struct timeval tv[2];
1147 
1148 		    bzero(tv, sizeof(tv));
1149 		    tv[0].tv_sec = st1.st_mtime;
1150 		    tv[1].tv_sec = st1.st_mtime;
1151 
1152 		    if (DstRootPrivs || ChgrpAllowed(st1.st_gid))
1153 			hc_chown(&DstHost, path, st1.st_uid, st1.st_gid);
1154 		    hc_chmod(&DstHost, path, st1.st_mode);
1155 #ifdef _ST_FLAGS_PRESENT_
1156 		    if (st1.st_flags & (UF_IMMUTABLE|SF_IMMUTABLE))
1157 			hc_utimes(&DstHost, path, tv);
1158 #else
1159 		    hc_utimes(&DstHost, path, tv);
1160 #endif
1161 		    if (st2Valid && xrename(path, dpath, st2_flags) != 0) {
1162 			logerr("%-32s rename-after-copy failed: %s\n",
1163 			    (dpath ? dpath : spath), strerror(errno)
1164 			);
1165 			++r;
1166 		    } else {
1167 			if (VerboseOpt)
1168 			    logstd("%-32s copy-ok\n", (dpath ? dpath : spath));
1169 #ifdef _ST_FLAGS_PRESENT_
1170 			if (DstRootPrivs ? st1.st_flags : st1.st_flags & UF_SETTABLE)
1171 			    hc_chflags(&DstHost, dpath, st1.st_flags);
1172 #endif
1173 		    }
1174 #ifdef _ST_FLAGS_PRESENT_
1175 		    if ((st1.st_flags & (UF_IMMUTABLE|SF_IMMUTABLE)) == 0)
1176 			hc_utimes(&DstHost, dpath, tv);
1177 #endif
1178 		    CountSourceReadBytes += size;
1179 		    CountWriteBytes += size;
1180 		    CountSourceBytes += size;
1181 		    CountSourceItems++;
1182 		    CountCopiedItems++;
1183 		} else {
1184 		    logerr("%-32s %s failed: %s\n",
1185 			(dpath ? dpath : spath), op, strerror(errno)
1186 		    );
1187 		    hc_remove(&DstHost, path);
1188 		    ++r;
1189 		}
1190 		free(iobuf1);
1191 	    } else {
1192 		logerr("%-32s create (uid %d, euid %d) failed: %s\n",
1193 		    (dpath ? dpath : spath), getuid(), geteuid(),
1194 		    strerror(errno)
1195 		);
1196 		++r;
1197 	    }
1198 	    hc_close(&SrcHost, fd1);
1199 	} else {
1200 	    logerr("%-32s copy: open failed: %s\n",
1201 		(dpath ? dpath : spath),
1202 		strerror(errno)
1203 	    );
1204 	    ++r;
1205 	}
1206 skip_copy:
1207 	free(path);
1208 
1209         if (hln) {
1210             if (!r && hc_stat(&DstHost, dpath, &st2) == 0) {
1211 		hltsetdino(hln, st2.st_ino);
1212 	    } else {
1213                 hltdelete(hln);
1214 		hln = NULL;
1215 	    }
1216         }
1217     } else if (S_ISLNK(st1.st_mode)) {
1218 	char *link1 = malloc(GETLINKSIZE);
1219 	char *link2 = malloc(GETLINKSIZE);
1220 	char *path;
1221 	int n1;
1222 	int n2;
1223 
1224 	n1 = hc_readlink(&SrcHost, spath, link1, GETLINKSIZE - 1);
1225 	if (st2Valid) {
1226 		path = mprintf("%s.tmp%d", dpath, (int)getpid());
1227 		n2 = hc_readlink(&DstHost, dpath, link2, GETLINKSIZE - 1);
1228 	} else {
1229 		path = mprintf("%s", dpath);
1230 		n2 = -1;
1231 	}
1232 	if (n1 >= 0) {
1233 	    if (ForceOpt || n1 != n2 || bcmp(link1, link2, n1) != 0) {
1234 		hc_umask(&DstHost, ~st1.st_mode);
1235 		xremove(&DstHost, path);
1236 		link1[n1] = 0;
1237 		if (hc_symlink(&DstHost, link1, path) < 0) {
1238                       logerr("%-32s symlink (%s->%s) failed: %s\n",
1239 			  (dpath ? dpath : spath), link1, path,
1240 			  strerror(errno)
1241 		      );
1242 		      ++r;
1243 		} else {
1244 		    if (DstRootPrivs || ChgrpAllowed(st1.st_gid))
1245 			hc_lchown(&DstHost, path, st1.st_uid, st1.st_gid);
1246 		    /*
1247 		     * there is no lchmod() or lchflags(), we
1248 		     * cannot chmod or chflags a softlink.
1249 		     */
1250 		    if (st2Valid && xrename(path, dpath, st2_flags) != 0) {
1251 			logerr("%-32s rename softlink (%s->%s) failed: %s\n",
1252 			    (dpath ? dpath : spath),
1253 			    path, dpath, strerror(errno));
1254 		    } else if (VerboseOpt) {
1255 			logstd("%-32s softlink-ok\n", (dpath ? dpath : spath));
1256 		    }
1257 		    hc_umask(&DstHost, 000);
1258 		    CountWriteBytes += n1;
1259 		    CountCopiedItems++;
1260 	  	}
1261 	    } else {
1262 		if (VerboseOpt >= 3)
1263 		    logstd("%-32s nochange", (dpath ? dpath : spath));
1264 		if (!OwnerMatch(&st1, &st2)) {
1265 		    hc_lchown(&DstHost, dpath, st1.st_uid, st1.st_gid);
1266 		    if (VerboseOpt >= 3)
1267 			logstd(" (uid/gid differ)");
1268 		}
1269 		if (VerboseOpt >= 3)
1270 		    logstd("\n");
1271 	    }
1272 	    CountSourceBytes += n1;
1273 	    CountSourceReadBytes += n1;
1274 	    if (n2 > 0)
1275 		CountTargetReadBytes += n2;
1276 	    CountSourceItems++;
1277 	} else {
1278 	    r = 1;
1279 	    logerr("%-32s softlink-failed\n", (dpath ? dpath : spath));
1280 	}
1281 	free(link1);
1282 	free(link2);
1283 	free(path);
1284     } else if ((S_ISCHR(st1.st_mode) || S_ISBLK(st1.st_mode)) && DeviceOpt) {
1285 	char *path = NULL;
1286 
1287 	if (ForceOpt ||
1288 	    st2Valid == 0 ||
1289 	    st1.st_mode != st2.st_mode ||
1290 	    st1.st_rdev != st2.st_rdev ||
1291 	    !OwnerMatch(&st1, &st2)
1292 	) {
1293 	    if (st2Valid) {
1294 		path = mprintf("%s.tmp%d", dpath, (int)getpid());
1295 		xremove(&DstHost, path);
1296 	    } else {
1297 		path = mprintf("%s", dpath);
1298 	    }
1299 
1300 	    if (hc_mknod(&DstHost, path, st1.st_mode, st1.st_rdev) == 0) {
1301 		hc_chmod(&DstHost, path, st1.st_mode);
1302 		hc_chown(&DstHost, path, st1.st_uid, st1.st_gid);
1303 		if (st2Valid)
1304 			xremove(&DstHost, dpath);
1305 		if (st2Valid && xrename(path, dpath, st2_flags) != 0) {
1306 		    logerr("%-32s dev-rename-after-create failed: %s\n",
1307 			(dpath ? dpath : spath),
1308 			strerror(errno)
1309 		    );
1310 		} else if (VerboseOpt) {
1311 		    logstd("%-32s dev-ok\n", (dpath ? dpath : spath));
1312 		}
1313 		CountCopiedItems++;
1314 	    } else {
1315 		r = 1;
1316 		logerr("%-32s dev failed: %s\n",
1317 		    (dpath ? dpath : spath), strerror(errno)
1318 		);
1319 	    }
1320 	} else {
1321 	    if (VerboseOpt >= 3)
1322 		logstd("%-32s nochange\n", (dpath ? dpath : spath));
1323 	}
1324 	if (path)
1325 		free(path);
1326 	CountSourceItems++;
1327     }
1328 done:
1329     if (hln) {
1330 	if (hln->dino == (ino_t)-1) {
1331 	    hltdelete(hln);
1332 	    /*hln = NULL; unneeded */
1333 	} else {
1334 	    hltrels(hln);
1335 	}
1336     }
1337     return (r);
1338 }
1339 
1340 int
1341 ScanDir(List *list, struct HostConf *host, const char *path,
1342 	int64_t *CountReadBytes, int n)
1343 {
1344     DIR *dir;
1345     struct dirent *den;
1346 
1347     if ((dir = hc_opendir(host, path)) == NULL)
1348 	return (1);
1349 
1350     if (n == 0) {
1351 	/*
1352 	 * scan .cpignore file for files/directories to ignore
1353 	 * (only in the source directory, i.e. if n == 0).
1354 	 */
1355 	if (UseCpFile) {
1356 	    int fd;
1357 	    int nread;
1358 	    int bufused;
1359 	    char *buf = malloc(GETBUFSIZE);
1360 	    char *nl, *next;
1361 	    char *fpath;
1362 
1363 	    if (UseCpFile[0] == '/') {
1364 		fpath = mprintf("%s", UseCpFile);
1365 	    } else {
1366 		fpath = mprintf("%s/%s", path, UseCpFile);
1367 	    }
1368 	    AddList(list, strrchr(fpath, '/') + 1, 1);
1369 	    if ((fd = hc_open(host, fpath, O_RDONLY, 0)) >= 0) {
1370 		bufused = 0;
1371 		while ((nread = hc_read(host, fd, buf + bufused,
1372 			GETIOSIZE - bufused - 1)) > 0) {
1373 		    *CountReadBytes += nread;
1374 		    bufused += nread;
1375 		    buf[bufused] = 0;
1376 		    for (next = buf; (nl = strchr(next, '\n')); next = nl+1) {
1377 			*nl = 0;
1378 			AddList(list, next, 1);
1379 		    }
1380 		    bufused = strlen(next);
1381 		    if (bufused)
1382 			bcopy(next, buf, bufused);
1383 		}
1384 		if (bufused) {
1385 		    /* last line has no trailing newline */
1386 		    buf[bufused] = 0;
1387 		    AddList(list, buf, 1);
1388 		}
1389 		hc_close(host, fd);
1390 	    }
1391 	    free(fpath);
1392 	    free(buf);
1393 	}
1394 
1395 	/*
1396 	 * Automatically exclude MD5CacheFile that we create on the
1397 	 * source from the copy to the destination.
1398 	 *
1399 	 * Automatically exclude a FSMIDCacheFile on the source that
1400 	 * would otherwise overwrite the one we maintain on the target.
1401 	 */
1402 	if (UseMD5Opt)
1403 	    AddList(list, MD5CacheFile, 1);
1404 	if (UseFSMIDOpt)
1405 	    AddList(list, FSMIDCacheFile, 1);
1406     }
1407 
1408     while ((den = hc_readdir(host, dir)) != NULL) {
1409 	/*
1410 	 * ignore . and ..
1411 	 */
1412 	if (strcmp(den->d_name, ".") != 0 && strcmp(den->d_name, "..") != 0)
1413 	     AddList(list, den->d_name, n);
1414     }
1415     hc_closedir(host, dir);
1416 
1417     return (0);
1418 }
1419 
1420 /*
1421  * RemoveRecur()
1422  */
1423 
1424 void
1425 RemoveRecur(const char *dpath, dev_t devNo)
1426 {
1427     struct stat st;
1428 
1429     if (hc_lstat(&DstHost, dpath, &st) == 0) {
1430 	if (devNo == (dev_t)-1)
1431 	    devNo = st.st_dev;
1432 	if (st.st_dev == devNo) {
1433 	    if (S_ISDIR(st.st_mode)) {
1434 		DIR *dir;
1435 
1436 		if ((dir = hc_opendir(&DstHost, dpath)) != NULL) {
1437 		    struct dirent *den;
1438 		    while ((den = hc_readdir(&DstHost, dir)) != NULL) {
1439 			char *ndpath;
1440 
1441 			if (strcmp(den->d_name, ".") == 0)
1442 			    continue;
1443 			if (strcmp(den->d_name, "..") == 0)
1444 			    continue;
1445 			ndpath = mprintf("%s/%s", dpath, den->d_name);
1446 			RemoveRecur(ndpath, devNo);
1447 			free(ndpath);
1448 		    }
1449 		    hc_closedir(&DstHost, dir);
1450 		}
1451 		if (AskConfirmation && NoRemoveOpt == 0) {
1452 		    if (YesNo(dpath)) {
1453 			if (hc_rmdir(&DstHost, dpath) < 0) {
1454 			    logerr("%-32s rmdir failed: %s\n",
1455 				dpath, strerror(errno)
1456 			    );
1457 			}
1458 			CountRemovedItems++;
1459 		    }
1460 		} else {
1461 		    if (NoRemoveOpt) {
1462 			if (VerboseOpt)
1463 			    logstd("%-32s not-removed\n", dpath);
1464 		    } else if (hc_rmdir(&DstHost, dpath) == 0) {
1465 			if (VerboseOpt)
1466 			    logstd("%-32s rmdir-ok\n", dpath);
1467 			CountRemovedItems++;
1468 		    } else {
1469 			logerr("%-32s rmdir failed: %s\n",
1470 			    dpath, strerror(errno)
1471 			);
1472 		    }
1473 		}
1474 	    } else {
1475 		if (AskConfirmation && NoRemoveOpt == 0) {
1476 		    if (YesNo(dpath)) {
1477 			if (xremove(&DstHost, dpath) < 0) {
1478 			    logerr("%-32s remove failed: %s\n",
1479 				dpath, strerror(errno)
1480 			    );
1481 			}
1482 			CountRemovedItems++;
1483 		    }
1484 		} else {
1485 		    if (NoRemoveOpt) {
1486 			if (VerboseOpt)
1487 			    logstd("%-32s not-removed\n", dpath);
1488 		    } else if (xremove(&DstHost, dpath) == 0) {
1489 			if (VerboseOpt)
1490 			    logstd("%-32s remove-ok\n", dpath);
1491 			CountRemovedItems++;
1492 		    } else {
1493 			logerr("%-32s remove failed: %s\n",
1494 			    dpath, strerror(errno)
1495 			);
1496 		    }
1497 		}
1498 	    }
1499 	}
1500     }
1501 }
1502 
1503 void
1504 InitList(List *list)
1505 {
1506     bzero(list, sizeof(List));
1507     list->li_Node.no_Next = &list->li_Node;
1508 }
1509 
1510 void
1511 ResetList(List *list)
1512 {
1513     Node *node;
1514 
1515     while ((node = list->li_Node.no_Next) != &list->li_Node) {
1516 	list->li_Node.no_Next = node->no_Next;
1517 	free(node);
1518     }
1519     InitList(list);
1520 }
1521 
1522 char *
1523 IterateList(List *list, Node **nodeptr, int n)
1524 {
1525     Node *node = *nodeptr;
1526 
1527     if (node == NULL)
1528 	node = list->li_Node.no_Next;
1529     while (node != &list->li_Node) {
1530 	if (node->no_Value == n) {
1531 	    *nodeptr = node->no_Next;
1532 	    return (node->no_Name);
1533 	}
1534 	node = node->no_Next;
1535     }
1536     return (NULL);
1537 }
1538 
1539 int
1540 AddList(List *list, const char *name, int n)
1541 {
1542     Node *node;
1543     int hv;
1544 
1545     hv = shash(name);
1546 
1547     /*
1548      * Scan against wildcards.  Only a node value of 1 can be a wildcard
1549      * ( usually scanned from .cpignore )
1550      */
1551 
1552     for (node = list->li_Hash[0]; node; node = node->no_HNext) {
1553 	if (strcmp(name, node->no_Name) == 0 ||
1554 	    (n != 1 && node->no_Value == 1 &&
1555 	    fnmatch(node->no_Name, name, 0) == 0)
1556 	) {
1557 	    return(node->no_Value);
1558 	}
1559     }
1560 
1561     /*
1562      * Look for exact match
1563      */
1564 
1565     for (node = list->li_Hash[hv]; node; node = node->no_HNext) {
1566 	if (strcmp(name, node->no_Name) == 0) {
1567 	    return(node->no_Value);
1568 	}
1569     }
1570     node = malloc(sizeof(Node) + strlen(name) + 1);
1571     if (node == NULL) {
1572         fprintf(stderr, "out of memory\n");
1573         exit(EXIT_FAILURE);
1574     }
1575 
1576     node->no_Next = list->li_Node.no_Next;
1577     list->li_Node.no_Next = node;
1578 
1579     node->no_HNext = list->li_Hash[hv];
1580     list->li_Hash[hv] = node;
1581 
1582     strcpy(node->no_Name, name);
1583     node->no_Value = n;
1584 
1585     return(n);
1586 }
1587 
1588 static int
1589 shash(const char *s)
1590 {
1591     int hv;
1592 
1593     hv = 0xA4FB3255;
1594 
1595     while (*s) {
1596 	if (*s == '*' || *s == '?' ||
1597 	    *s == '{' || *s == '}' ||
1598 	    *s == '[' || *s == ']' ||
1599 	    *s == '|'
1600 	) {
1601 	    return(0);
1602 	}
1603 	hv = (hv << 5) ^ *s ^ (hv >> 23);
1604 	++s;
1605     }
1606     return(((hv >> 16) ^ hv) & HMASK);
1607 }
1608 
1609 int
1610 YesNo(const char *path)
1611 {
1612     int ch, first;
1613 
1614     fprintf(stderr, "remove %s (Yes/No) [No]? ", path);
1615     fflush(stderr);
1616 
1617     first = ch = getchar();
1618     while (ch != '\n' && ch != EOF)
1619 	ch = getchar();
1620     return ((first == 'y' || first == 'Y'));
1621 }
1622 
1623 /*
1624  * xrename() - rename with override
1625  *
1626  *	If the rename fails, attempt to override st_flags on the
1627  *	destination and rename again.  If that fails too, try to
1628  *	set the flags back the way they were and give up.
1629  */
1630 
1631 static int
1632 xrename(const char *src, const char *dst, u_long flags)
1633 {
1634     int r;
1635 
1636     if ((r = hc_rename(&DstHost, src, dst)) < 0) {
1637 #ifdef _ST_FLAGS_PRESENT_
1638 	hc_chflags(&DstHost, dst, 0);
1639 	if ((r = hc_rename(&DstHost, src, dst)) < 0)
1640 		hc_chflags(&DstHost, dst, flags);
1641 #endif
1642     }
1643     return(r);
1644 }
1645 
1646 static int
1647 xlink(const char *src, const char *dst, u_long flags)
1648 {
1649     int r;
1650 #ifdef _ST_FLAGS_PRESENT_
1651     int e;
1652 #endif
1653 
1654     if ((r = hc_link(&DstHost, src, dst)) < 0) {
1655 #ifdef _ST_FLAGS_PRESENT_
1656 	hc_chflags(&DstHost, src, 0);
1657 	r = hc_link(&DstHost, src, dst);
1658 	e = errno;
1659 	hc_chflags(&DstHost, src, flags);
1660 	errno = e;
1661 #endif
1662     }
1663     if (r == 0)
1664 	    ++CountLinkedItems;
1665     return(r);
1666 }
1667 
1668 static int
1669 xremove(struct HostConf *host, const char *path)
1670 {
1671     int res;
1672 
1673     res = hc_remove(host, path);
1674 #ifdef _ST_FLAGS_PRESENT_
1675     if (res == -EPERM) {
1676 	hc_chflags(host, path, 0);
1677 	res = hc_remove(host, path);
1678     }
1679 #endif
1680     return(res);
1681 }
1682 
1683