1 /*
2  * lftp - file transfer program
3  *
4  * Copyright (c) 1996-2017 by Alexander V. Lukyanov (lav@yars.free.net)
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include <config.h>
21 
22 #include <sys/types.h>
23 #include <sys/stat.h>
24 #include <errno.h>
25 #include <fcntl.h>
26 #include <unistd.h>
27 #include <assert.h>
28 #include <mbswidth.h>
29 #include "MirrorJob.h"
30 #include "CmdExec.h"
31 #include "rmJob.h"
32 #include "mvJob.h"
33 #include "ChmodJob.h"
34 #include "mkdirJob.h"
35 #include "misc.h"
36 #include "plural.h"
37 #include "FindJob.h"
38 #include "url.h"
39 #include "CopyJob.h"
40 #include "pgetJob.h"
41 #include "log.h"
42 
43 #define set_state(s) do { state=(s); \
44    Log::global->Format(11,"mirror(%p) enters state %s\n", this, #s); } while(0)
45 #define waiting_num waiting.count()
46 #define transfer_count root_mirror->root_transfer_count
47 
FormatStatus(xstring & s,int v,const char * tab)48 xstring& MirrorJob::FormatStatus(xstring& s,int v,const char *tab)
49 {
50    if(Done())
51       goto final;
52 
53    switch(state)
54    {
55    case(INITIAL_STATE):
56    case(FINISHING):
57    case(DONE):
58    case(WAITING_FOR_TRANSFER):
59    case(TARGET_REMOVE_OLD):
60    case(TARGET_REMOVE_OLD_FIRST):
61    case(TARGET_CHMOD):
62    case(TARGET_MKDIR):
63    case(SOURCE_REMOVING_SAME):
64    case(LAST_EXEC):
65       break;
66 
67    case(MAKE_TARGET_DIR):
68       s.appendf("\tmkdir `%s' [%s]\n",target_dir.get(),target_session->CurrentStatus());
69       break;
70 
71    case(CHANGING_DIR_SOURCE):
72    case(CHANGING_DIR_TARGET):
73       if(target_session->IsOpen())
74 	 s.appendf("\tcd `%s' [%s]\n",target_dir.get(),target_session->CurrentStatus());
75       if(source_session->IsOpen())
76 	 s.appendf("\tcd `%s' [%s]\n",source_dir.get(),source_session->CurrentStatus());
77       break;
78 
79    case(GETTING_LIST_INFO):
80       if(target_list_info)
81       {
82 	 if(target_relative_dir)
83 	    s.appendf("\t%s: %s\n",target_relative_dir.get(),target_list_info->Status());
84 	 else
85 	    s.appendf("\t%s\n",target_list_info->Status());
86       }
87       if(source_list_info)
88       {
89 	 if(source_relative_dir)
90 	    s.appendf("\t%s: %s\n",source_relative_dir.get(),source_list_info->Status());
91 	 else
92 	    s.appendf("\t%s\n",source_list_info->Status());
93       }
94       break;
95    }
96    return s;
97 
98 final:
99    if(stats.dirs>0)
100       s.appendf(plural("%sTotal: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n",
101 		     stats.dirs,stats.tot_files,stats.tot_symlinks),
102 	 tab,stats.dirs,stats.tot_files,stats.tot_symlinks);
103    if(stats.new_files || stats.new_symlinks)
104       s.appendf(plural("%sNew: %d file$|s$, %d symlink$|s$\n",
105 		     stats.new_files,stats.new_symlinks),
106 	 tab,stats.new_files,stats.new_symlinks);
107    if(stats.mod_files || stats.mod_symlinks)
108       s.appendf(plural("%sModified: %d file$|s$, %d symlink$|s$\n",
109 		     stats.mod_files,stats.mod_symlinks),
110 	 tab,stats.mod_files,stats.mod_symlinks);
111    if(stats.bytes)
112       s.appendf("%s%s\n",tab,CopyJob::FormatBytesTimeRate(stats.bytes,transfer_time_elapsed));
113    if(stats.del_dirs || stats.del_files || stats.del_symlinks)
114       s.appendf(plural(FlagSet(DELETE) ?
115 	       "%sRemoved: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n"
116 	      :"%sTo be removed: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n",
117 	      stats.del_dirs,stats.del_files,stats.del_symlinks),
118 	 tab,stats.del_dirs,stats.del_files,stats.del_symlinks);
119    if(stats.error_count)
120       s.appendf(plural("%s%d error$|s$ detected\n",stats.error_count),
121 	       tab,stats.error_count);
122    return s;
123 }
124 
ShowRunStatus(const SMTaskRef<StatusLine> & s)125 void  MirrorJob::ShowRunStatus(const SMTaskRef<StatusLine>& s)
126 {
127    int w=s->GetWidthDelayed();
128    switch(state)
129    {
130    case(INITIAL_STATE):
131       break;
132 
133    // these have a sub-job
134    case(WAITING_FOR_TRANSFER):
135    case(TARGET_REMOVE_OLD):
136    case(TARGET_REMOVE_OLD_FIRST):
137    case(TARGET_CHMOD):
138    case(TARGET_MKDIR):
139    case(SOURCE_REMOVING_SAME):
140    case(FINISHING):
141    case(DONE):
142    case(LAST_EXEC):
143       Job::ShowRunStatus(s);
144       break;
145 
146    case(MAKE_TARGET_DIR):
147       s->Show("mkdir `%s' [%s]",target_dir.get(),target_session->CurrentStatus());
148       break;
149 
150    case(CHANGING_DIR_SOURCE):
151    case(CHANGING_DIR_TARGET):
152       if(target_session->IsOpen() && (!source_session->IsOpen() || now%4>=2))
153 	 s->Show("cd `%s' [%s]",target_dir.get(),target_session->CurrentStatus());
154       else if(source_session->IsOpen())
155 	 s->Show("cd `%s' [%s]",source_dir.get(),source_session->CurrentStatus());
156       break;
157 
158    case(GETTING_LIST_INFO):
159       if(target_list_info && (!source_list_info || now%4>=2))
160       {
161 	 const char *status=target_list_info->Status();
162 	 int status_w=mbswidth(status, 0);
163 	 int dw=w-status_w;
164 	 if(dw<20)
165 	    dw=20;
166 	 if(target_relative_dir)
167 	    s->Show("%s: %s",squeeze_file_name(target_relative_dir,dw),status);
168 	 else
169 	    s->Show("%s",status);
170       }
171       else if(source_list_info)
172       {
173 	 const char *status=source_list_info->Status();
174 	 int status_w=mbswidth(status, 0);
175 	 int dw=w-status_w;
176 	 if(dw<20)
177 	    dw=20;
178 	 if(source_relative_dir)
179 	    s->Show("%s: %s",squeeze_file_name(source_relative_dir,dw),status);
180 	 else
181 	    s->Show("%s",status);
182       }
183       break;
184    }
185 }
186 
FormatShortStatus(xstring & s)187 xstring& MirrorJob::FormatShortStatus(xstring& s)
188 {
189    if(bytes_to_transfer>0 && (!parent_mirror || parent_mirror->bytes_to_transfer!=bytes_to_transfer)) {
190       long long curr_bytes_transferred=GetBytesCount();
191       if(parent_mirror)
192          curr_bytes_transferred+=bytes_transferred;
193       s.appendf("%s/%s (%d%%)",
194 	 xhuman(curr_bytes_transferred),xhuman(bytes_to_transfer),
195 	 percent(curr_bytes_transferred,bytes_to_transfer));
196       double rate=GetTransferRate();
197       if(rate>=1)
198 	 s.append(' ').append(Speedometer::GetStrProper(rate));
199    }
200    return s;
201 }
202 
TransferStarted(CopyJob * cp)203 void MirrorJob::TransferStarted(CopyJob *cp)
204 {
205    if(transfer_count==0)
206       root_mirror->transfer_start_ts=now;
207    JobStarted(cp);
208 }
JobStarted(Job * j)209 void MirrorJob::JobStarted(Job *j)
210 {
211    AddWaiting(j);
212    transfer_count++;
213 }
TransferFinished(Job * j)214 void MirrorJob::TransferFinished(Job *j)
215 {
216    long long bytes_count=j->GetBytesCount();
217    AddBytesTransferred(bytes_count);
218    stats.bytes+=bytes_count;
219    stats.time +=j->GetTimeSpent();
220    if(j->ExitCode()==0 && verbose_report>=2) {
221       xstring finished;
222       const xstring& cmd=j->GetCmdLine();
223       if(cmd[0]=='\\')
224 	 finished.append(cmd+1,cmd.length()-1);
225       else
226 	 finished.append(cmd);
227       const xstring& rate=Speedometer::GetStrProper(j->GetTransferRate());
228       if(rate.length()>0)
229 	 finished.append(" (").append(rate).append(')');
230       if(!(FlagSet(SCAN_ALL_FIRST) && finished.begins_with("mirror")))
231 	 Report(_("Finished %s"),finished.get());
232    }
233    JobFinished(j);
234    if(transfer_count==0)
235       root_mirror->transfer_time_elapsed += now-root_mirror->transfer_start_ts;
236 }
JobFinished(Job * j)237 void MirrorJob::JobFinished(Job *j)
238 {
239    if(j->ExitCode()!=0)
240       stats.error_count++;
241    RemoveWaiting(j);
242    Delete(j);
243    assert(transfer_count>0);
244    transfer_count--;
245 }
246 
GetBytesCount()247 off_t MirrorJob::GetBytesCount()
248 {
249    long long bytes_count=Job::GetBytesCount();
250    if(!parent_mirror) {
251       // bytes_transferred is cumulative over the mirror tree,
252       // add it on the top only
253       bytes_count+=bytes_transferred;
254    }
255    return bytes_count;
256 }
GetTimeSpent()257 double MirrorJob::GetTimeSpent()
258 {
259    double t=transfer_time_elapsed;
260    if(transfer_count>0)
261       t+=now-root_mirror->transfer_start_ts;
262    return t;
263 }
264 
HandleFile(FileInfo * file)265 void  MirrorJob::HandleFile(FileInfo *file)
266 {
267    int	 res;
268    struct stat st;
269 
270    // TODO: get rid of local hacks.
271 
272    const char *dst_name=file->name;
273    if(FlagSet(TARGET_FLAT))
274       dst_name=basename_ptr(dst_name);
275 
276    // dir_name returns pointer to static data - need to dup it.
277    const char *source_name=dir_file(source_dir,file->name);
278    source_name=alloca_strdup(source_name);
279    const char *target_name=dir_file(target_dir,dst_name);
280    target_name=alloca_strdup(target_name);
281 
282    const char *source_name_rel=dir_file(source_relative_dir,file->name);
283    source_name_rel=alloca_strdup(source_name_rel);
284    const char *target_name_rel=dir_file(target_relative_dir,dst_name);
285    target_name_rel=alloca_strdup(target_name_rel);
286 
287    FileInfo::type filetype=FileInfo::NORMAL;
288    if(file->Has(file->TYPE))
289       filetype=file->filetype;
290    else
291    {
292       FileInfo *target=target_set->FindByName(file->name);
293       if(target && target->Has(target->TYPE))
294 	 filetype=target->filetype;
295    }
296 
297    switch(filetype)
298    {
299       case(FileInfo::NORMAL):
300       case(FileInfo::REDIRECT):
301       {
302 	 bool remove_target=false;
303 	 bool cont_this=false;
304 	 bool use_pget=(pget_n>1) && target_is_local;
305 	 if(file->Has(file->SIZE) && file->size<pget_minchunk*2)
306 	    use_pget=false;
307 	 if(target_is_local)
308 	 {
309 	    if(lstat(target_name,&st)!=-1)
310 	    {
311 	       // few safety checks.
312 	       FileInfo *old=new_files_set->FindByName(file->name);
313 	       if(old)
314 		  goto skip;  // file has appeared after mirror start
315 	       old=old_files_set->FindByName(file->name);
316 	       if(old && ((old->Has(old->SIZE) && old->size!=st.st_size)
317 			||(old->Has(old->DATE) && old->date!=st.st_mtime)))
318 		  goto skip;  // the file has changed after mirror start
319 	       if(!script_only && access(target_name,W_OK)==-1)
320 	       {
321 		  // try to enable write access.
322 		  chmod(target_name,st.st_mode|0200);
323 	       }
324 	    }
325 	 }
326 	 FileInfo *old=target_set->FindByName(FileCopy::TempFileName(file->name));
327 	 if(old)
328 	 {
329 	    if(FlagSet(CONTINUE)
330 	    && old->Has(file->TYPE) && old->filetype==old->NORMAL
331 	    && (FlagSet(IGNORE_TIME) ||
332 		(file->Has(file->DATE) && old->Has(old->DATE)
333 		&& file->date + file->date.ts_prec < old->date - old->date.ts_prec))
334 	    && file->Has(file->SIZE) && old->Has(old->SIZE)
335 	    && file->size >= old->size)
336 	    {
337 	       cont_this=true;
338 	       stats.mod_files++;
339 	    }
340 	    else if(!to_rm_mismatched->FindByName(file->name))
341 	    {
342 	       if(!FlagSet(OVERWRITE)) {
343 		  remove_target=true;
344 		  Report(_("Removing old file `%s'"),target_name_rel);
345 	       } else {
346 		  Report(_("Overwriting old file `%s'"),target_name_rel);
347 	       }
348 	       stats.mod_files++;
349 	    }
350 	    else
351 	       stats.new_files++;
352 	 }
353 	 else if(FlagSet(ONLY_EXISTING))
354 	 {
355 	    Report(_("Skipping file `%s' (only-existing)"),source_name_rel);
356 	    goto skip;
357 	 }
358 	 else
359 	    stats.new_files++;
360 
361 	 Report(_("Transferring file `%s'"),source_name_rel);
362 
363 	 if(script)
364 	 {
365 	    ArgV args(use_pget?"pget":"get");
366 	    if(use_pget)
367 	    {
368 	       args.Append("-n");
369 	       args.Append(pget_n);
370 	    }
371 	    if(cont_this)
372 	       args.Append("-c");
373 	    if(remove_target)
374 	       args.Append("-e");
375 	    if(FlagSet(ASCII))
376 	       args.Append("-a");
377 	    if(remove_source_files)
378 	       args.Append("-E");
379 	    args.Append("-O");
380 	    args.Append(target_is_local?target_dir.get()
381 			:target_session->GetConnectURL().get());
382 	    args.Append(source_session->GetFileURL(file->name));
383 	    xstring_ca cmd(args.CombineQuoted());
384 	    fprintf(script,"%s\n",cmd.get());
385 	    if(script_only)
386 	       goto skip;
387 	 }
388 
389 	 FileCopyPeer *src_peer=0;
390 	 if(source_is_local)
391 	    src_peer=new FileCopyPeerFDStream(new FileStream(source_name,O_RDONLY),FileCopyPeer::GET);
392 	 else
393 	    src_peer=new FileCopyPeerFA(source_session->Clone(),file->name,FA::RETRIEVE);
394 
395 	 FileCopyPeer *dst_peer=0;
396 	 if(target_is_local)
397 	    dst_peer=new FileCopyPeerFDStream(new FileStream(target_name,O_WRONLY|O_CREAT|(cont_this?0:O_TRUNC)),FileCopyPeer::PUT);
398 	 else
399 	    dst_peer=new FileCopyPeerFA(target_session->Clone(),dst_name,FA::STORE);
400 
401 	 FileCopy *c=FileCopy::New(src_peer,dst_peer,cont_this);
402 	 if(remove_source_files)
403 	    c->RemoveSourceLater();
404 	 if(remove_target)
405 	    c->RemoveTargetFirst();
406 	 if(FlagSet(ASCII))
407 	    c->Ascii();
408 	 CopyJob *cp=(use_pget ? new pgetJob(c,file->name,pget_n) : new CopyJob(c,file->name,"mirror"));
409 	 if(file->Has(file->DATE))
410 	    cp->SetDate(file->date);
411 	 if(file->Has(file->SIZE) && !FlagSet(IGNORE_SIZE))
412 	    cp->SetSize(file->size);
413 	 TransferStarted(cp);
414 	 cp->cmdline.vset("\\transfer `",source_name_rel,"'",NULL);
415 
416 	 set_state(WAITING_FOR_TRANSFER);
417 	 break;
418       }
419       case(FileInfo::DIRECTORY):
420       {
421 	 if(recursion_mode==RECURSION_NEVER || FlagSet(NO_RECURSION))
422 	    goto skip;
423 
424 	 bool create_target_subdir=true;
425 	 const FileInfo *old=0;
426 
427 	 if(FlagSet(TARGET_FLAT)) {
428 	    create_target_subdir=false;
429 	    target_name=target_dir;
430 	    goto do_submirror;
431 	 }
432 
433 	 if(target_set)
434 	    old=target_set->FindByName(file->name);
435 	 if(!old)
436 	 {
437 	    if(FlagSet(ONLY_EXISTING))
438 	    {
439 	       Report(_("Skipping directory `%s' (only-existing)"),target_name_rel);
440 	       goto skip;
441 	    }
442 	 }
443 	 else if(old->TypeIs(old->DIRECTORY))
444 	 {
445 	    create_target_subdir=false;
446 	 }
447 	 if(target_is_local && !script_only)
448 	 {
449 	    if((FlagSet(RETR_SYMLINKS)?stat:lstat)(target_name,&st)!=-1)
450 	    {
451 	       if(S_ISDIR(st.st_mode))
452 	       {
453 		  // try to enable write access
454 		  // only if not enabled as chmod can clear sgid flags on directories
455 		  if(st.st_mode!=(st.st_mode|0700))
456 		     chmod(target_name,st.st_mode|0700);
457 		  create_target_subdir=false;
458 	       }
459 	       else
460 	       {
461 		  Report(_("Removing old local file `%s'"),target_name_rel);
462 		  if(remove(target_name)==-1)
463 		  {
464 		     eprintf("mirror: remove(%s): %s\n",target_name,strerror(errno));
465 		     goto skip;
466 		  }
467 		  create_target_subdir=true;
468 	       }
469 	    }
470 	 }
471 
472       do_submirror:
473 	 // launch sub-mirror
474 	 MirrorJob *mj=new MirrorJob(this,
475 	    source_session->Clone(),target_session->Clone(),
476 	    source_name,target_name);
477 	 AddWaiting(mj);
478 	 mj->cmdline.vset("\\mirror `",source_name_rel,"'",NULL);
479 
480 	 mj->source_relative_dir.set(source_name_rel);
481 	 mj->target_relative_dir.set(target_name_rel);
482 
483 	 mj->create_target_dir=create_target_subdir;
484 
485 	 if(verbose_report>=3) {
486 	    if(FlagSet(SCAN_ALL_FIRST))
487 	       Report(_("Scanning directory `%s'"),mj->target_relative_dir.get());
488 	    else
489 	       Report(_("Mirroring directory `%s'"),mj->target_relative_dir.get());
490 	 }
491 
492 	 break;
493       }
494       case(FileInfo::SYMLINK):
495       {
496 	 if(FlagSet(NO_SYMLINKS))
497 	    goto skip;
498 
499 	 if(!file->symlink)
500 	    goto skip;
501 
502 	 if(!target_is_local)
503 	 {
504 	    if(script)
505 	    {
506 	       ArgV args("ln");
507 	       args.Append("-s");
508 	       args.Append(file->symlink);
509 	       args.Append(target_name);
510 	       xstring_ca cmd(args.CombineQuoted());
511 	       fprintf(script,"%s\n",cmd.get());
512 	       if(script_only)
513 		  goto skip;
514 	    }
515 	    bool remove_target=false;
516 	    FileInfo *old=target_set->FindByName(file->name);
517 	    if(old && !to_rm_mismatched->FindByName(file->name))
518 	    {
519 	       Report(_("Removing old file `%s'"),target_name_rel);
520 	       remove_target=true;
521 	       stats.mod_symlinks++;
522 	    }
523 	    else
524 	       stats.new_symlinks++;
525 	    Report(_("Making symbolic link `%s' to `%s'"),target_name_rel,file->symlink.get());
526 	    mvJob *j=new mvJob(target_session->Clone(),file->symlink,target_name,FA::SYMLINK);
527 	    if(remove_target)
528 	       j->RemoveTargetFirst();
529 	    JobStarted(j);
530 	    RemoveSourceLater(file);
531 	    break;
532 	 }
533 
534 	 if(script)
535 	 {
536 	    ArgV args("shell");
537 	    args.Append("ln");
538 	    args.Append("-sf");
539 	    args.Append(shell_encode(file->symlink));
540 	    args.Append(shell_encode(target_name));
541 	    xstring_ca cmd(args.CombineQuoted());
542 	    fprintf(script,"%s\n",cmd.get());
543 	    if(script_only)
544 	       goto skip;
545 	 }
546 
547 	 struct stat st;
548 	 if(lstat(target_name,&st)!=-1)
549 	 {
550 	    Report(_("Removing old local file `%s'"),target_name_rel);
551 	    stats.mod_symlinks++;
552 	    if(remove(target_name)==-1)
553 	    {
554 	       eprintf("mirror: remove(%s): %s\n",target_name,strerror(errno));
555 	       goto skip;
556 	    }
557 	 }
558 	 else
559 	 {
560 	    if(FlagSet(ONLY_EXISTING))
561 	    {
562 	       Report(_("Skipping symlink `%s' (only-existing)"),target_name_rel);
563 	       goto skip;
564 	    }
565 	    stats.new_symlinks++;
566 	 }
567 	 Report(_("Making symbolic link `%s' to `%s'"),target_name_rel,file->symlink.get());
568 	 res=symlink(file->symlink,target_name);
569 	 if(res==-1)
570 	    eprintf("mirror: symlink(%s): %s\n",target_name,strerror(errno));
571 	 RemoveSourceLater(file);
572 	 break;
573       }
574    case FileInfo::UNKNOWN:
575       break;
576    }
577 skip:
578    return;
579 }
580 
InitSets()581 void  MirrorJob::InitSets()
582 {
583    if(FlagSet(TARGET_FLAT) && !parent_mirror && target_set)
584       source_set->Sort(FileSet::BYNAME_FLAT);
585 
586    source_set->Count(NULL,&stats.tot_files,&stats.tot_symlinks,&stats.tot_files);
587 
588    to_rm=new FileSet(target_set);
589    to_rm->SubtractAny(source_set);
590 
591    if(FlagSet(DELETE_EXCLUDED) && target_set_excluded)
592       to_rm->Merge(target_set_excluded);
593 
594    to_transfer=new FileSet(source_set);
595 
596    if(!FlagSet(TRANSFER_ALL)) {
597       same=new FileSet(source_set);
598 
599       int ignore=0;
600       if(FlagSet(ONLY_NEWER))
601 	 ignore|=FileInfo::IGNORE_SIZE_IF_OLDER|FileInfo::IGNORE_DATE_IF_OLDER;
602       if(!FlagSet(UPLOAD_OLDER) && strcmp(target_session->GetProto(),"file"))
603 	 ignore|=FileInfo::IGNORE_DATE_IF_OLDER;
604       if(FlagSet(IGNORE_TIME))
605 	 ignore|=FileInfo::DATE;
606       if(FlagSet(IGNORE_SIZE))
607 	 ignore|=FileInfo::SIZE;
608       to_transfer->SubtractSame(target_set,ignore);
609 
610       same->SubtractAny(to_transfer);
611    }
612 
613    if(newer_than!=NO_DATE)
614       to_transfer->SubtractNotNewerThan(newer_than);
615    if(older_than!=NO_DATE)
616       to_transfer->SubtractNotOlderThan(older_than);
617    if(size_range)
618       to_transfer->SubtractSizeOutside(size_range);
619 
620    if(FlagSet(SCAN_ALL_FIRST)) {
621       to_mkdir=new FileSet(to_transfer);
622       to_mkdir->SubtractNotDirs();
623       to_mkdir->SubtractAny(target_set);
624    }
625 
626    switch(recursion_mode) {
627    case RECURSION_NEVER:
628       to_transfer->SubtractDirs();
629       break;
630    case RECURSION_MISSING:
631       to_transfer->SubtractDirs(target_set);
632       break;
633    case RECURSION_NEWER:
634       to_transfer->SubtractNotOlderDirs(target_set);
635       break;
636    case RECURSION_ALWAYS:
637       break;
638    }
639 
640    if(skip_noaccess)
641       to_transfer->ExcludeUnaccessible(source_session->GetUser());
642 
643    new_files_set=new FileSet(to_transfer);
644    new_files_set->SubtractAny(target_set);
645    old_files_set=new FileSet(target_set);
646    old_files_set->SubtractNotIn(to_transfer);
647 
648    to_rm_mismatched=new FileSet(old_files_set);
649    to_rm_mismatched->SubtractSameType(to_transfer);
650    to_rm_mismatched->SubtractNotDirs();
651 
652    if(!FlagSet(DELETE))
653       to_transfer->SubtractAny(to_rm_mismatched);
654 
655    if(FlagSet(TARGET_FLAT) && !parent_mirror && target_set) {
656       source_set->Unsort();
657       to_transfer->UnsortFlat();
658       to_transfer->SubtractDirs();
659       same->UnsortFlat();
660       to_mkdir->Empty();
661       new_files_set->UnsortFlat();
662    }
663 
664    const char *sort_by=ResMgr::Query("mirror:sort-by",0);
665    bool desc=strstr(sort_by,"-desc");
666    if(!strncmp(sort_by,"name",4))
667       to_transfer->SortByPatternList(ResMgr::Query("mirror:order",0));
668    else if(!strncmp(sort_by,"date",4))
669       to_transfer->Sort(FileSet::BYDATE);
670    else if(!strncmp(sort_by,"size",4))
671       to_transfer->Sort(FileSet::BYSIZE,false,true);
672    if(desc)
673       to_transfer->ReverseSort();
674 
675    int dir_count=0;
676    if(to_mkdir) {
677       to_mkdir->Count(&dir_count,NULL,NULL,NULL);
678       only_dirs = (dir_count==to_mkdir->count());
679    } else {
680       to_transfer->Count(&dir_count,NULL,NULL,NULL);
681       only_dirs = (dir_count==to_transfer->count());
682    }
683 }
684 
ExcludeEmptyDir(const char * target_rel_dir)685 void MirrorJob::ExcludeEmptyDir(const char *target_rel_dir)
686 {
687    source_set->SubtractByName(basename_ptr(target_rel_dir));
688 }
689 
690 /* root_transfer_count of child mirrors contains the value to add or
691    subtract from transfer_count when doing "cd", "mkdir", "ls" on the source
692    or target directories. This would prevent other mirrors (siblings, etc)
693    from starting more jobs.
694 
695    root_transfer_count is initialized once in ctor, so that change of
696    mirror:parallel-directories setting won't disbalance the count.
697 */
MirrorStarted()698 void MirrorJob::MirrorStarted()
699 {
700    if(!parent_mirror)
701       return;
702    transfer_count+=root_transfer_count;
703 }
MirrorFinished()704 void MirrorJob::MirrorFinished()
705 {
706    if(!parent_mirror)
707       return;
708    assert(transfer_count>=root_transfer_count);
709    transfer_count-=root_transfer_count;
710 }
711 
HandleChdir(FileAccessRef & session,int & redirections)712 void MirrorJob::HandleChdir(FileAccessRef& session, int &redirections)
713 {
714    if(!session->IsOpen())
715       return;
716    int res=session->Done();
717    if(res<0)
718    {
719       if(res==FA::NO_HOST)
720       {
721 	 eprintf("mirror: %s\n",session->StrError(res));
722 	 stats.error_count++;
723 	 MirrorFinished();
724 	 set_state(FINISHING);
725 	 return;
726       }
727       if(res==FA::FILE_MOVED)
728       {
729 	 // cd to another url.
730 	 const char *loc_c=session->GetNewLocation();
731 	 int max_redirections=ResMgr::Query("xfer:max-redirections",0);
732 	 if(loc_c && max_redirections>0)
733 	 {
734 	    if(++redirections>max_redirections)
735 	       goto cd_err_normal;
736 	    eprintf(_("%s: received redirection to `%s'\n"),"mirror",loc_c);
737 
738 	    char *loc=alloca_strdup(loc_c);
739 	    ParsedURL u(loc,true);
740 
741 	    bool is_file=(last_char(loc)!='/');
742 	    if(!u.proto)
743 	    {
744 	       FileAccess::Path new_cwd(session->GetNewCwd());
745 	       new_cwd.Change(0,is_file,loc);
746 	       session->PathVerify(new_cwd);
747 	       session->Roll();
748 	       return;
749 	    }
750 	    session->Close(); // loc_c is no longer valid.
751 	    session=FA::New(&u);
752 	    FileAccess::Path new_cwd(u.path,is_file,url::path_ptr(loc));
753 	    session->PathVerify(new_cwd);
754 	    return;
755 	 }
756       }
757    cd_err_normal:
758       if(session==target_session && (script_only || FlagSet(SCAN_ALL_FIRST)))
759       {
760 	 char *dir=alloca_strdup(session->GetFile());
761 	 session->Close();
762 	 session->Chdir(dir,false);
763 	 no_target_dir=true;
764 	 return;
765       }
766       if(session==source_session && create_target_dir
767       && !FlagSet(NO_EMPTY_DIRS) && !skip_noaccess && parent_mirror)
768       {
769 	 // create target dir even if failed to cd to source dir.
770 	 if(script)
771 	    fprintf(script,"mkdir %s\n",target_session->GetFileURL(target_dir).get());
772 	 if(!script_only)
773 	 {
774 	    ArgV *a=new ArgV("mkdir");
775 	    a->Append(target_dir);
776 	    mkdirJob *mkj=new mkdirJob(target_session->Clone(),a);
777 	    a->CombineTo(mkj->cmdline);
778 	    JobStarted(mkj);
779 	 }
780       }
781       remove_this_source_dir=false;
782       eprintf("mirror: %s\n",session->StrError(res));
783       stats.error_count++;
784       MirrorFinished();
785       set_state(FINISHING);
786       source_session->Close();
787       target_session->Close();
788       return;
789    }
790    if(res==FA::OK)
791       session->Close();
792 }
HandleListInfoCreation(const FileAccessRef & session,SMTaskRef<ListInfo> & list_info,const char * relative_dir)793 void MirrorJob::HandleListInfoCreation(const FileAccessRef& session,SMTaskRef<ListInfo>& list_info,const char *relative_dir)
794 {
795    if(state!=GETTING_LIST_INFO)
796       return;
797 
798    if(session==target_session && no_target_dir)
799    {
800       target_set=new FileSet();
801       return;
802    }
803 
804    list_info=session->MakeListInfo();
805    if(list_info==0)
806    {
807       eprintf(_("mirror: protocol `%s' is not suitable for mirror\n"),
808 	       session->GetProto());
809       MirrorFinished();
810       set_state(FINISHING);
811       return;
812    }
813    list_info->UseCache(use_cache);
814    int need=FileInfo::ALL_INFO;
815    if(FlagSet(IGNORE_TIME))
816       need&=~FileInfo::DATE;
817    if(FlagSet(IGNORE_SIZE))
818       need&=~FileInfo::SIZE;
819    list_info->Need(need);
820    if(FlagSet(RETR_SYMLINKS))
821       list_info->FollowSymlinks();
822 
823    list_info->SetExclude(relative_dir,top_exclude?top_exclude:exclude);
824    list_info->Roll();
825 }
826 
HandleListInfo(SMTaskRef<ListInfo> & list_info,Ref<FileSet> & set,Ref<FileSet> * fsx)827 void MirrorJob::HandleListInfo(SMTaskRef<ListInfo>& list_info, Ref<FileSet>& set, Ref<FileSet> *fsx)
828 {
829    if(!list_info)
830       return;
831    if(!list_info->Done())
832       return;
833    if(list_info->Error())
834    {
835       eprintf("mirror: %s\n",list_info->ErrorText());
836       stats.error_count++;
837       MirrorFinished();
838       set_state(FINISHING);
839       source_list_info=0;
840       target_list_info=0;
841       return;
842    }
843    set=list_info->GetResult();
844    if(fsx)
845    {
846       *fsx=list_info->GetExcluded();
847       (*fsx)->ExcludeDots();
848    }
849    list_info=0;
850    set->ExcludeDots(); // don't need .. and .
851 }
852 
Do()853 int   MirrorJob::Do()
854 {
855    int	 res;
856    int	 m=STALL;
857    FileInfo *file;
858    Job	 *j;
859 
860    switch(state)
861    {
862    case(INITIAL_STATE):
863       remove_this_source_dir=(remove_source_dirs && source_dir.last_char()!='/');
864       if(!strcmp(target_dir,".") || !strcmp(target_dir,"..") || (FlagSet(SCAN_ALL_FIRST) && parent_mirror))
865 	 create_target_dir=false;
866 
867       source_session->Chdir(source_dir);
868       source_redirections=0;
869       source_session->Roll();
870       set_state(CHANGING_DIR_SOURCE);
871       m=MOVED;
872       /*fallthrough*/
873    case(CHANGING_DIR_SOURCE):
874       HandleChdir(source_session,source_redirections);
875       if(state!=CHANGING_DIR_SOURCE)
876 	 return MOVED;
877       if(source_session->IsOpen())
878 	 return m;
879 
880       source_dir.set(source_session->GetCwd().GetDirectory());
881 
882    pre_MAKE_TARGET_DIR:
883    {
884       if(!create_target_dir)
885 	 goto pre_CHANGING_DIR_TARGET;
886       if(target_is_local)
887       {
888 	 struct stat st;
889 	 if((FlagSet(RETR_SYMLINKS)?stat:lstat)(target_dir,&st)!=-1)
890 	 {
891 	    if(S_ISDIR(st.st_mode))
892 	    {
893 	       // try to enable write access
894 	       // only if not enabled as chmod can clear sgid flags on directories
895 	       if(!script_only && (st.st_mode!=(st.st_mode|0700)))
896 		  chmod(target_dir,st.st_mode|0700);
897 	       create_target_dir=false;
898 	       goto pre_CHANGING_DIR_TARGET;
899 	    }
900 	    else
901 	    {
902 	       Report(_("Removing old local file `%s'"),target_dir.get());
903 	       if(script)
904 	       {
905 		  ArgV args("rm");
906 		  args.Append(target_session->GetFileURL(target_dir));
907 		  xstring_ca cmd(args.CombineQuoted());
908 		  fprintf(script,"%s\n",cmd.get());
909 	       }
910 	       if(!script_only)
911 	       {
912 		  if(remove(target_dir)==-1)
913 		     eprintf("mirror: remove(%s): %s\n",target_dir.get(),strerror(errno));
914 	       }
915 	    }
916 	 }
917       }
918 
919       if(FlagSet(DEPTH_FIRST))
920 	 goto pre_GETTING_LIST_INFO;
921 
922       if(target_relative_dir)
923 	 Report(_("Making directory `%s'"),target_relative_dir.get());
924       bool mkdir_p=(parent_mirror==0 || parent_mirror->create_target_dir);
925       if(script)
926       {
927 	 ArgV args("mkdir");
928 	 if(mkdir_p)
929 	    args.Append("-p");
930 	 args.Append(target_session->GetFileURL(target_dir));
931 	 xstring_ca cmd(args.CombineQuoted());
932 	 fprintf(script,"%s\n",cmd.get());
933 	 if(script_only)
934 	    goto pre_CHANGING_DIR_TARGET;
935       }
936       target_session->Mkdir(target_dir,mkdir_p);
937       set_state(MAKE_TARGET_DIR);
938       m=MOVED;
939    }
940       /*fallthrough*/
941    case(MAKE_TARGET_DIR):
942       res=target_session->Done();
943       if(res==FA::IN_PROGRESS)
944 	 return m;
945       target_session->Close();
946       create_target_dir=false;
947 
948    pre_CHANGING_DIR_TARGET:
949       target_session->Chdir(target_dir);
950       target_redirections=0;
951       target_session->Roll();
952       set_state(CHANGING_DIR_TARGET);
953       m=MOVED;
954       /*fallthrough*/
955    case(CHANGING_DIR_TARGET):
956       HandleChdir(target_session,target_redirections);
957       if(state!=CHANGING_DIR_TARGET)
958 	 return MOVED;
959       if(target_session->IsOpen())
960 	 return m;
961       create_target_dir=false;
962 
963       target_dir.set(target_session->GetCwd().GetDirectory());
964 
965    pre_GETTING_LIST_INFO:
966       set_state(GETTING_LIST_INFO);
967       m=MOVED;
968       if(!source_set)
969 	 HandleListInfoCreation(source_session,source_list_info,source_relative_dir);
970       if(!target_set && !create_target_dir
971       && (!FlagSet(DEPTH_FIRST) || FlagSet(ONLY_EXISTING))
972       && !(FlagSet(TARGET_FLAT) && parent_mirror))
973 	 HandleListInfoCreation(target_session,target_list_info,target_relative_dir);
974       if(state!=GETTING_LIST_INFO)
975       {
976 	 source_list_info=0;
977 	 target_list_info=0;
978       }
979       return m;	  // give time to other tasks
980    case(GETTING_LIST_INFO):
981       HandleListInfo(source_list_info,source_set);
982       HandleListInfo(target_list_info,target_set,&target_set_excluded);
983       if(state!=GETTING_LIST_INFO)
984 	 return MOVED;
985       if(source_list_info || target_list_info)
986 	 return m;
987 
988       MirrorFinished(); // leave room for transfers.
989 
990       if(FlagSet(DEPTH_FIRST) && source_set && !target_set)
991       {
992 	 // transfer directories first
993 	 InitSets();
994 	 to_transfer->Unsort();
995 	 to_transfer->SubtractNotDirs();
996 	 goto pre_WAITING_FOR_TRANSFER;
997       }
998 
999       // now we have both target and source file sets.
1000       if(parent_mirror)
1001 	 stats.dirs++;
1002 
1003       if(FlagSet(SCAN_ALL_FIRST) && parent_mirror)
1004       {
1005 	 source_set->PrependPath(source_relative_dir);
1006 	 if(root_mirror->source_set_recursive)
1007 	    root_mirror->source_set_recursive->Merge(source_set);
1008 	 else
1009 	    root_mirror->source_set_recursive=source_set.borrow();
1010 	 if(target_set) {
1011 	    target_set->PrependPath(target_relative_dir);
1012 	    if(root_mirror->target_set_recursive)
1013 	       root_mirror->target_set_recursive->Merge(target_set);
1014 	    else
1015 	       root_mirror->target_set_recursive=target_set.borrow();
1016 	 }
1017 	 if(target_set_excluded) {
1018 	    target_set_excluded->PrependPath(target_relative_dir);
1019 	    if(root_mirror->target_set_excluded)
1020 	       root_mirror->target_set_excluded->Merge(target_set_excluded);
1021 	    else
1022 	       root_mirror->target_set_excluded=target_set_excluded.borrow();
1023 	 }
1024 	 root_mirror->stats.dirs++;
1025 	 transfer_count++; // parent mirror will decrement it.
1026 	 goto pre_DONE;
1027       }
1028 
1029       if(source_set_recursive) {
1030 	 source_set->Merge(source_set_recursive);
1031 	 source_set_recursive=0;
1032       }
1033       if(target_set_recursive) {
1034 	 target_set->Merge(target_set_recursive);
1035 	 target_set_recursive=0;
1036       }
1037       InitSets();
1038 
1039       to_transfer->CountBytes(&bytes_to_transfer);
1040       if(parent_mirror)
1041 	 parent_mirror->AddBytesToTransfer(bytes_to_transfer);
1042 
1043       to_rm->Count(&stats.del_dirs,&stats.del_files,&stats.del_symlinks,&stats.del_files);
1044       to_rm->rewind();
1045       to_rm_mismatched->Count(&stats.del_dirs,&stats.del_files,&stats.del_symlinks,&stats.del_files);
1046       to_rm_mismatched->rewind();
1047 
1048       target_set->Merge(target_set_excluded);
1049       target_set_excluded=0;
1050 
1051       set_state(TARGET_REMOVE_OLD_FIRST);
1052       goto TARGET_REMOVE_OLD_FIRST_label;
1053 
1054    pre_TARGET_MKDIR:
1055       if(!to_mkdir)
1056 	 goto pre_WAITING_FOR_TRANSFER;
1057       to_mkdir->rewind();
1058       set_state(TARGET_MKDIR);
1059       m=MOVED;
1060       /*fallthrough*/
1061    case(TARGET_MKDIR):
1062       while((j=FindDoneAwaitedJob())!=0)
1063       {
1064 	 JobFinished(j);
1065 	 m=MOVED;
1066       }
1067       if(max_error_count>0 && stats.error_count>=max_error_count)
1068 	 goto pre_FINISHING;
1069       while(transfer_count<parallel && state==TARGET_MKDIR)
1070       {
1071 	 file=to_mkdir->curr();
1072 	 if(!file)
1073 	    goto pre_WAITING_FOR_TRANSFER;
1074 	 to_mkdir->next();
1075 	 if(!file->TypeIs(file->DIRECTORY))
1076 	    continue;
1077 	 if(script)
1078 	    fprintf(script,"mkdir %s\n",target_session->GetFileURL(file->name).get());
1079 	 if(!script_only)
1080 	 {
1081 	    ArgV *a=new ArgV("mkdir");
1082 	    a->Append(file->name);
1083 	    mkdirJob *mkj=new mkdirJob(target_session->Clone(),a);
1084 	    a->CombineTo(mkj->cmdline);
1085 	    JobStarted(mkj);
1086 	    m=MOVED;
1087 	 }
1088       }
1089       break;
1090 
1091    pre_WAITING_FOR_TRANSFER:
1092       to_transfer->rewind();
1093       set_state(WAITING_FOR_TRANSFER);
1094       m=MOVED;
1095       /*fallthrough*/
1096    case(WAITING_FOR_TRANSFER):
1097       while((j=FindDoneAwaitedJob())!=0)
1098       {
1099 	 TransferFinished(j);
1100 	 m=MOVED;
1101       }
1102       if(max_error_count>0 && stats.error_count>=max_error_count)
1103 	 goto pre_FINISHING;
1104       while(transfer_count<parallel && state==WAITING_FOR_TRANSFER)
1105       {
1106 	 file=to_transfer->curr();
1107 	 if(!file)
1108 	 {
1109 	    // go to the next step only when all transfers have finished
1110 	    if(waiting_num>0)
1111 	       break;
1112 	    if(FlagSet(DEPTH_FIRST))
1113 	    {
1114 	       // we have been in the depth, don't go there again
1115 	       SetFlags(DEPTH_FIRST,false);
1116 	       SetFlags(NO_RECURSION,true);
1117 
1118 	       // if we have not created any subdirs and there are only subdirs,
1119 	       // then the directory would be empty - skip it.
1120 	       if(FlagSet(NO_EMPTY_DIRS) && stats.dirs==0 && only_dirs)
1121 	       {
1122 		  if(parent_mirror)
1123 		     parent_mirror->ExcludeEmptyDir(target_relative_dir);
1124 		  goto pre_FINISHING_FIX_LOCAL;
1125 	       }
1126 
1127 	       MirrorStarted();
1128 	       goto pre_MAKE_TARGET_DIR;
1129 	    }
1130 	    goto pre_TARGET_REMOVE_OLD;
1131 	 }
1132 	 HandleFile(file);
1133 	 to_transfer->next();
1134 	 m=MOVED;
1135       }
1136       break;
1137 
1138    pre_TARGET_REMOVE_OLD:
1139       if(FlagSet(REMOVE_FIRST))
1140 	 goto pre_TARGET_CHMOD;
1141       set_state(TARGET_REMOVE_OLD);
1142       m=MOVED;
1143       /*fallthrough*/
1144    case(TARGET_REMOVE_OLD):
1145    case(TARGET_REMOVE_OLD_FIRST):
1146    TARGET_REMOVE_OLD_FIRST_label:
1147       while((j=FindDoneAwaitedJob())!=0)
1148       {
1149 	 JobFinished(j);
1150 	 m=MOVED;
1151       }
1152       if(max_error_count>0 && stats.error_count>=max_error_count)
1153 	 goto pre_FINISHING;
1154       while(transfer_count<parallel && (state==TARGET_REMOVE_OLD || state==TARGET_REMOVE_OLD_FIRST))
1155       {
1156 	 file=0;
1157 	 if(!file && state==TARGET_REMOVE_OLD_FIRST)
1158 	 {
1159 	    file=to_rm_mismatched->curr();
1160 	    to_rm_mismatched->next();
1161 	 }
1162 	 if(!file && (state==TARGET_REMOVE_OLD || FlagSet(REMOVE_FIRST)))
1163 	 {
1164 	    file=to_rm->curr();
1165 	    to_rm->next();
1166 	 }
1167 	 if(!file)
1168 	 {
1169 	    if(waiting_num>0)
1170 	       break;
1171 	    if(state==TARGET_REMOVE_OLD)
1172 	       goto pre_TARGET_CHMOD;
1173 	    goto pre_TARGET_MKDIR;
1174 	 }
1175 	 if(!FlagSet(DELETE))
1176 	 {
1177 	    if(FlagSet(REPORT_NOT_DELETED))
1178 	    {
1179 	       const char *target_name_rel=dir_file(target_relative_dir,file->name);
1180 	       if(file->TypeIs(file->DIRECTORY))
1181 		  Report(_("Old directory `%s' is not removed"),target_name_rel);
1182 	       else
1183 		  Report(_("Old file `%s' is not removed"),target_name_rel);
1184 	    }
1185 	    continue;
1186 	 }
1187 	 bool use_rmdir = (file->TypeIs(file->DIRECTORY)
1188 			   && recursion_mode==RECURSION_NEVER);
1189 	 if(script)
1190 	 {
1191 	    ArgV args(use_rmdir?"rmdir":"rm");
1192 	    if(file->TypeIs(file->DIRECTORY) && !use_rmdir)
1193 	       args.Append("-r");
1194 	    args.Append(target_session->GetFileURL(file->name));
1195 	    xstring_ca cmd(args.CombineQuoted());
1196 	    fprintf(script,"%s\n",cmd.get());
1197 	 }
1198 	 if(!script_only)
1199 	 {
1200 	    ArgV *args=new ArgV(use_rmdir?"rmdir":"rm");
1201 	    args->Append(dir_file(".",file->name));
1202 	    args->seek(1);
1203 	    rmJob *j=new rmJob(target_session->Clone(),args);
1204 	    args->CombineTo(j->cmdline);
1205 	    JobStarted(j);
1206 	    if(file->TypeIs(file->DIRECTORY))
1207 	    {
1208 	       if(recursion_mode==RECURSION_NEVER)
1209 		  j->Rmdir();
1210 	       else
1211 		  j->Recurse();
1212 	    }
1213 	 }
1214 	 const char *target_name_rel=dir_file(target_relative_dir,file->name);
1215 	 if(file->TypeIs(file->DIRECTORY))
1216 	    Report(_("Removing old directory `%s'"),target_name_rel);
1217 	 else
1218 	    Report(_("Removing old file `%s'"),target_name_rel);
1219       }
1220       break;
1221 
1222    pre_TARGET_CHMOD:
1223       if(FlagSet(NO_PERMS))
1224 	 goto pre_FINISHING_FIX_LOCAL;
1225 
1226       to_transfer->rewind();
1227       if(FlagSet(TARGET_FLAT))
1228 	 to_transfer->Sort(FileSet::BYNAME_FLAT);
1229       set_state(TARGET_CHMOD);
1230       m=MOVED;
1231       /*fallthrough*/
1232    case(TARGET_CHMOD):
1233       while((j=FindDoneAwaitedJob())!=0)
1234       {
1235 	 JobFinished(j);
1236 	 m=MOVED;
1237       }
1238       if(max_error_count>0 && stats.error_count>=max_error_count)
1239 	 goto pre_FINISHING;
1240       while(transfer_count<parallel && state==TARGET_CHMOD)
1241       {
1242 	 file=to_transfer->curr();
1243 	 if(!file)
1244 	    goto pre_FINISHING_FIX_LOCAL;
1245 	 to_transfer->next();
1246 	 if(file->TypeIs(file->SYMLINK))
1247 	    continue;
1248 	 if(!file->Has(file->MODE))
1249 	    continue;
1250 	 mode_t mode_mask=get_mode_mask();
1251 	 mode_t def_mode=(file->TypeIs(file->DIRECTORY)?0775:0664)&~mode_mask;
1252 	 if(target_is_local && file->mode==def_mode)
1253 	 {
1254 	    struct stat st;
1255 	    if(!target_is_local || lstat(dir_file(target_dir,file->name),&st)==-1)
1256 	       continue;
1257 	    if((st.st_mode&07777)==(file->mode&~mode_mask))
1258 	       continue;
1259 	 }
1260 	 FileInfo *target=target_set->FindByName(file->name);
1261 	 if(target && target->filetype==file->DIRECTORY && file->filetype==file->DIRECTORY
1262 	 && target->mode==(file->mode&~mode_mask) && (target->mode&0200))
1263 	    continue;
1264 	 if(script)
1265 	 {
1266 	    ArgV args("chmod");
1267 	    args.Append("-f");
1268 	    args.Append(xstring::format("%03lo",(unsigned long)(file->mode&~mode_mask)));
1269 	    args.Append(target_session->GetFileURL(file->name));
1270 	    xstring_ca cmd(args.CombineQuoted());
1271 	    fprintf(script,"%s\n",cmd.get());
1272 	 }
1273 	 if(!script_only)
1274 	 {
1275 	    ArgV *a=new ArgV("chmod");
1276 	    a->Append(dir_file(".",file->name));
1277 	    a->seek(1);
1278 	    ChmodJob *cj=new ChmodJob(target_session->Clone(),
1279 				 file->mode&~mode_mask,a);
1280 	    a->CombineTo(cj->cmdline);
1281 	    if(!verbose_report)
1282 	       cj->BeQuiet(); // chmod is not supported on all servers; be quiet.
1283 	    JobStarted(cj);
1284 	    m=MOVED;
1285 	 }
1286       }
1287       break;
1288 
1289    pre_FINISHING_FIX_LOCAL:
1290       if(target_is_local && !script_only)     // FIXME
1291       {
1292 	 const bool flat=FlagSet(TARGET_FLAT);
1293 	 to_transfer->Sort(FileSet::BYNAME_FLAT);
1294 	 to_transfer->LocalUtime(target_dir,/*only_dirs=*/true,flat);
1295 	 if(FlagSet(ALLOW_CHOWN))
1296 	    to_transfer->LocalChown(target_dir,flat);
1297 	 if(!FlagSet(NO_PERMS) && same)
1298 	    same->LocalChmod(target_dir,get_mode_mask(),flat);
1299 	 if(FlagSet(ALLOW_CHOWN) && same)
1300 	    same->LocalChown(target_dir,flat);
1301       }
1302       if(remove_source_files && (same || to_rm_src))
1303 	 goto pre_SOURCE_REMOVING_SAME;
1304    pre_FINISHING:
1305       set_state(FINISHING);
1306       m=MOVED;
1307       /*fallthrough*/
1308    case(FINISHING):
1309       while((j=FindDoneAwaitedJob())!=0)
1310       {
1311 	 JobFinished(j);
1312 	 m=MOVED;
1313       }
1314       if(waiting_num>0)
1315 	 break;
1316 
1317       // all jobs finished.
1318       if(remove_this_source_dir) {
1319 	 // remove source directory once.
1320 	 remove_this_source_dir=false;
1321 	 if(script)
1322 	 {
1323 	    ArgV args("rmdir");
1324 	    args.Append(source_session->GetFileURL(source_dir));
1325 	    xstring_ca cmd(args.CombineQuoted());
1326 	    fprintf(script,"%s\n",cmd.get());
1327 	 }
1328 	 if(!script_only)
1329 	 {
1330 	    ArgV *args=new ArgV("rmdir");
1331 	    args->Append(source_dir);
1332 	    args->seek(1);
1333 	    rmJob *j=new rmJob(source_session->Clone(),args);
1334 	    args->CombineTo(j->cmdline);
1335 	    j->Rmdir();
1336 	    JobStarted(j);
1337 	 }
1338 	 if(source_relative_dir)
1339 	    Report(_("Removing source directory `%s'"),source_relative_dir.get());
1340 	 m=MOVED;
1341 	 break;
1342       }
1343 
1344       // all jobs finished and src dir removed, if needed.
1345 
1346       transfer_count++; // parent mirror will decrement it.
1347       if(parent_mirror)
1348 	 parent_mirror->stats.Add(stats);
1349       else
1350       {
1351 	 if(stats.HaveSomethingDone(flags) && on_change)
1352 	 {
1353 	    CmdExec *exec=new CmdExec(GetExecSession().Clone(),0);
1354 	    AddWaiting(exec);
1355 	    exec->FeedCmd(on_change);
1356 	    exec->FeedCmd("\n");
1357 	    set_state(LAST_EXEC);
1358 	    break;
1359 	 }
1360       }
1361       goto pre_DONE;
1362 
1363    pre_SOURCE_REMOVING_SAME:
1364       if(!same)
1365 	 same=to_rm_src.borrow();
1366       else if(to_rm_src)
1367 	 same->Merge(to_rm_src);
1368       same->rewind();
1369       set_state(SOURCE_REMOVING_SAME);
1370       m=MOVED;
1371       /*fallthrough*/
1372    case(SOURCE_REMOVING_SAME):
1373       while((j=FindDoneAwaitedJob())!=0)
1374       {
1375 	 JobFinished(j);
1376 	 m=MOVED;
1377       }
1378       if(max_error_count>0 && stats.error_count>=max_error_count)
1379 	 goto pre_FINISHING;
1380       while(transfer_count<parallel && state==SOURCE_REMOVING_SAME)
1381       {
1382 	 file=same->curr();
1383 	 same->next();
1384 	 if(!file)
1385 	    goto pre_FINISHING;
1386 	 if(file->TypeIs(file->DIRECTORY))
1387 	    continue;
1388 	 if(script)
1389 	 {
1390 	    ArgV args("rm");
1391 	    args.Append(source_session->GetFileURL(file->name));
1392 	    xstring_ca cmd(args.CombineQuoted());
1393 	    fprintf(script,"%s\n",cmd.get());
1394 	 }
1395 	 if(!script_only)
1396 	 {
1397 	    ArgV *args=new ArgV("rm");
1398 	    args->Append(dir_file(".",file->name));
1399 	    args->seek(1);
1400 	    rmJob *j=new rmJob(source_session->Clone(),args);
1401 	    args->CombineTo(j->cmdline);
1402 	    JobStarted(j);
1403 	 }
1404 	 const char *source_name_rel=dir_file(source_relative_dir,file->name);
1405 	 Report(_("Removing source file `%s'"),source_name_rel);
1406       }
1407       break;
1408 
1409    case(LAST_EXEC):
1410       while((j=FindDoneAwaitedJob())!=0)
1411       {
1412 	 RemoveWaiting(j);
1413 	 Delete(j);
1414 	 m=MOVED;
1415       }
1416       if(waiting_num>0)
1417 	 break;
1418    pre_DONE:
1419       set_state(DONE);
1420       m=MOVED;
1421       bytes_transferred=0;
1422       if(!parent_mirror && FlagSet(LOOP) && stats.HaveSomethingDone(flags) && !stats.error_count)
1423       {
1424 	 PrintStatus(0,"");
1425 	 printf(_("Retrying mirror...\n"));
1426 	 stats.Reset();
1427 	 source_set=0;
1428 	 target_set=0;
1429 	 goto pre_GETTING_LIST_INFO;
1430       }
1431       /*fallthrough*/
1432    case(DONE):
1433       break;
1434    }
1435    // give direct parent priority over grand-parents.
1436    if(transfer_count<parallel && parent_mirror)
1437       m|=parent_mirror->Roll();
1438    return m;
1439 }
1440 
MirrorJob(MirrorJob * parent,FileAccess * source,FileAccess * target,const char * new_source_dir,const char * new_target_dir)1441 MirrorJob::MirrorJob(MirrorJob *parent,
1442    FileAccess *source,FileAccess *target,
1443    const char *new_source_dir,const char *new_target_dir)
1444  :
1445    bytes_transferred(0), bytes_to_transfer(0),
1446    source_dir(new_source_dir), target_dir(new_target_dir),
1447    transfer_time_elapsed(0), root_transfer_count(0),
1448    verbose_report(0),
1449    parent_mirror(parent), root_mirror(parent?parent->root_mirror:this)
1450 {
1451 
1452    source_session=source;
1453    target_session=target;
1454    // TODO: get rid of this.
1455    source_is_local=!strcmp(source_session->GetProto(),"file");
1456    target_is_local=!strcmp(target_session->GetProto(),"file");
1457 
1458    create_target_dir=true;
1459    no_target_dir=false;
1460    remove_this_source_dir=false;
1461 
1462    flags=0;
1463    recursion_mode=RECURSION_ALWAYS;
1464    max_error_count=0;
1465 
1466    exclude=0;
1467 
1468    set_state(INITIAL_STATE);
1469 
1470    newer_than=NO_DATE;
1471    older_than=NO_DATE;
1472    size_range=0;
1473 
1474    script=0;
1475    script_only=false;
1476    script_needs_closing=false;
1477 
1478    use_cache=false;
1479    remove_source_files=false;
1480    remove_source_dirs=false;
1481    skip_noaccess=false;
1482 
1483    parallel=1;
1484    pget_n=1;
1485    pget_minchunk=0x10000;
1486 
1487    source_redirections=0;
1488    target_redirections=0;
1489 
1490    if(parent_mirror)
1491    {
1492       bool parallel_dirs=ResMgr::QueryBool("mirror:parallel-directories",0);
1493       // If parallel_dirs is true, allow parent mirror to continue
1494       // processing other directories, otherwise block it until we
1495       // get file sets and start transfers.
1496       // See also comment at MirrorJob::MirrorStarted().
1497       root_transfer_count=parallel_dirs?1:1024;
1498 
1499       // inherit flags and other things
1500       SetFlags(parent->flags,1);
1501       UseCache(parent->use_cache);
1502 
1503       SetExclude(parent->exclude);
1504 
1505       verbose_report=parent->verbose_report;
1506       newer_than=parent->newer_than;
1507       older_than=parent->older_than;
1508       size_range=parent->size_range;
1509       parallel=parent->parallel;
1510       pget_n=parent->pget_n;
1511       pget_minchunk=parent->pget_minchunk;
1512       remove_source_files=parent->remove_source_files;
1513       remove_source_dirs=parent->remove_source_dirs;
1514       skip_noaccess=parent->skip_noaccess;
1515       no_target_dir=parent->no_target_dir;
1516       recursion_mode=parent->recursion_mode;
1517 
1518       script=parent->script;
1519       script_needs_closing=false;
1520       script_name.set(parent->script_name);
1521       script_only=parent->script_only;
1522 
1523       max_error_count=parent->max_error_count;
1524    }
1525    MirrorStarted();
1526 }
1527 
~MirrorJob()1528 MirrorJob::~MirrorJob()
1529 {
1530    if(script && script_needs_closing)
1531       fclose(script);
1532 }
1533 
va_Report(const char * fmt,va_list v)1534 void MirrorJob::va_Report(const char *fmt,va_list v)
1535 {
1536    if(parent_mirror)
1537    {
1538       parent_mirror->va_Report(fmt,v);
1539       return;
1540    }
1541 
1542    if(verbose_report)
1543    {
1544       pid_t p=tcgetpgrp(fileno(stdout));
1545       if(p>0 && p!=getpgrp())
1546 	 return;
1547 
1548       vfprintf(stdout,fmt,v);
1549       printf("\n");
1550       fflush(stdout);
1551    }
1552 }
1553 
Report(const char * fmt,...)1554 void MirrorJob::Report(const char *fmt,...)
1555 {
1556    va_list v;
1557    va_start(v,fmt);
1558 
1559    va_Report(fmt,v);
1560 
1561    va_end(v);
1562 }
1563 
1564 extern "C" {
1565 #include "parse-datetime.h"
1566 }
SetNewerThan(const char * f)1567 void MirrorJob::SetNewerThan(const char *f)
1568 {
1569    struct timespec ts;
1570    if(parse_datetime(&ts,f,0))
1571    {
1572       newer_than=ts.tv_sec;
1573       return;
1574    }
1575    struct stat st;
1576    if(stat(f,&st)==-1)
1577    {
1578       perror(f);
1579       return;
1580    }
1581    newer_than=st.st_mtime;
1582 }
SetOlderThan(const char * f)1583 void MirrorJob::SetOlderThan(const char *f)
1584 {
1585    struct timespec ts;
1586    if(parse_datetime(&ts,f,0))
1587    {
1588       older_than=ts.tv_sec;
1589       return;
1590    }
1591    struct stat st;
1592    if(stat(f,&st)==-1)
1593    {
1594       perror(f);
1595       return;
1596    }
1597    older_than=st.st_mtime;
1598 }
1599 
get_mode_mask()1600 mode_t MirrorJob::get_mode_mask()
1601 {
1602    mode_t mode_mask=0;
1603    if(!FlagSet(ALLOW_SUID))
1604       mode_mask|=S_ISUID|S_ISGID;
1605    if(!FlagSet(NO_UMASK))
1606    {
1607       if(target_is_local)
1608       {
1609 	 mode_t u=umask(022); // get+set
1610 	 umask(u);	      // retore
1611 	 mode_mask|=u;
1612       }
1613       else
1614 	 mode_mask|=022;   // sane default.
1615    }
1616    return mode_mask;
1617 }
1618 
Fg()1619 void MirrorJob::Fg()
1620 {
1621    Job::Fg();
1622    source_session->SetPriority(1);
1623    target_session->SetPriority(1);
1624 }
Bg()1625 void MirrorJob::Bg()
1626 {
1627    source_session->SetPriority(0);
1628    target_session->SetPriority(0);
1629    Job::Bg();
1630 }
1631 
Statistics()1632 MirrorJob::Statistics::Statistics()
1633 {
1634    Reset();
1635    error_count=0;
1636    bytes=0;
1637    time=0;
1638 }
Reset()1639 void MirrorJob::Statistics::Reset()
1640 {
1641    tot_files=new_files=mod_files=del_files=
1642    tot_symlinks=new_symlinks=mod_symlinks=del_symlinks=
1643    dirs=del_dirs=0;
1644 }
Add(const Statistics & s)1645 void MirrorJob::Statistics::Add(const Statistics &s)
1646 {
1647    tot_files   +=s.tot_files;
1648    new_files   +=s.new_files;
1649    mod_files   +=s.mod_files;
1650    del_files   +=s.del_files;
1651    tot_symlinks+=s.tot_symlinks;
1652    new_symlinks+=s.new_symlinks;
1653    mod_symlinks+=s.mod_symlinks;
1654    del_symlinks+=s.del_symlinks;
1655    dirs        +=s.dirs;
1656    del_dirs    +=s.del_dirs;
1657    error_count +=s.error_count;
1658    bytes       +=s.bytes;
1659    time	       +=s.time;
1660 }
HaveSomethingDone(unsigned flags)1661 bool MirrorJob::Statistics::HaveSomethingDone(unsigned flags)
1662 {
1663    bool del=(flags&MirrorJob::DELETE);
1664    return new_files|mod_files|(del_files*del)|new_symlinks|mod_symlinks|(del_symlinks*del)|(del_dirs*del);
1665 }
1666 
SetScriptFile(const char * n)1667 const char *MirrorJob::SetScriptFile(const char *n)
1668 {
1669    script_name.set(n);
1670    if(strcmp(n,"-"))
1671    {
1672       script=fopen(n,"w");
1673       if(!script)
1674 	 return xstring::format("%s: %s",n,strerror(errno));
1675       setvbuf(script,NULL,_IOLBF,0);
1676       script_needs_closing=true;
1677    }
1678    else
1679    {
1680       script=stdout;
1681       script_needs_closing=false;
1682    }
1683    return 0;
1684 }
1685 
SetOnChange(const char * oc)1686 void MirrorJob::SetOnChange(const char *oc)
1687 {
1688    on_change.set(oc);
1689 }
1690 
AddPattern(Ref<PatternSet> & exclude,char opt,const char * optarg)1691 const char *MirrorJob::AddPattern(Ref<PatternSet>& exclude,char opt,const char *optarg)
1692 {
1693    if(!optarg || !*optarg)
1694       return _("pattern is empty");
1695 
1696    PatternSet::Type type=
1697       (opt=='x'||opt=='X'||opt=='\0'?PatternSet::EXCLUDE:PatternSet::INCLUDE);
1698    PatternSet::Pattern *pattern=0;
1699    if(opt=='x' || opt=='i')
1700    {
1701       Ref<PatternSet::Regex> rx(new PatternSet::Regex(optarg));
1702       if(rx->Error())
1703 	 return xstring::get_tmp(rx->ErrorText());
1704       pattern=rx.borrow();
1705    }
1706    else if(opt=='X' || opt=='I')
1707    {
1708       pattern=new PatternSet::Glob(optarg);
1709    }
1710    if(!exclude)
1711    {
1712       const char *default_exclude=ResMgr::Query("mirror:exclude-regex",0);
1713       const char *default_include=ResMgr::Query("mirror:include-regex",0);
1714 
1715       // don't create default pattern set if not needed
1716       if(!pattern && !(default_exclude && *default_exclude))
1717 	 return NULL;
1718 
1719       exclude=new PatternSet;
1720       /* Make default_exclude the first pattern so that it can be
1721        * overridden by --include later, and do that only when first
1722        * explicit pattern is for exclusion - otherwise all files are
1723        * excluded by default and no default exclusion is needed. */
1724       if(type==PatternSet::EXCLUDE && default_exclude && *default_exclude)
1725       {
1726 	 exclude->Add(type,new PatternSet::Regex(default_exclude));
1727 	 if(default_include && *default_include)
1728 	    exclude->Add(PatternSet::INCLUDE,new PatternSet::Regex(default_include));
1729       }
1730    }
1731    if(pattern)
1732       exclude->Add(type,pattern);
1733 
1734    return NULL; // no error
1735 }
1736 
AddPatternsFrom(Ref<PatternSet> & exclude,char opt,const char * file)1737 const char *MirrorJob::AddPatternsFrom(Ref<PatternSet>& exclude,char opt,const char *file)
1738 {
1739    FILE *f=fopen(file,"r");
1740    if(!f)
1741       return xstring::format("%s: %s",file,strerror(errno));
1742 
1743    xstring line;
1744    const char *err=0;
1745    int c;
1746    while(!feof(f)) {
1747       line.truncate();
1748       while((c=getc(f))!=EOF && c!='\n')
1749 	 line.append(c);
1750       if(line.length()>0) {
1751 	 err=AddPattern(exclude,opt,line);
1752 	 if(err)
1753 	    break;
1754       }
1755    }
1756 
1757    fclose(f);
1758    return err;
1759 }
1760 
SetRecursionMode(const char * m)1761 const char *MirrorJob::SetRecursionMode(const char *m)
1762 {
1763    struct { const char name[8]; recursion_mode_t mode; } map[]={
1764       {"always", RECURSION_ALWAYS},
1765       {"never",  RECURSION_NEVER},
1766       {"missing",RECURSION_MISSING},
1767       {"newer",  RECURSION_NEWER},
1768    };
1769    unsigned i;
1770    for(i=0; i<sizeof(map)/sizeof(map[0]); i++) {
1771       if(!strcasecmp(m,map[i].name)) {
1772 	 recursion_mode=map[i].mode;
1773 	 return 0;
1774       }
1775    }
1776    xstring list(map[0].name);
1777    for(i=1; i<sizeof(map)/sizeof(map[0]); i++)
1778       list.append(", ").append(map[i].name);
1779    return xstring::format(_("%s must be one of: %s"),"--recursion",list.get());
1780 }
1781 
CMD(mirror)1782 CMD(mirror)
1783 {
1784 #define args (parent->args)
1785 #define eprintf parent->eprintf
1786    enum {
1787       OPT_ALLOW_CHOWN,
1788       OPT_DELETE_FIRST,
1789       OPT_IGNORE_SIZE,
1790       OPT_IGNORE_TIME,
1791       OPT_LOOP,
1792       OPT_MAX_ERRORS,
1793       OPT_NO_DEREFERENCE,
1794       OPT_NO_SYMLINKS,
1795       OPT_NO_UMASK,
1796       OPT_OLDER_THAN,
1797       OPT_ONLY_MISSING,
1798       OPT_ONLY_EXISTING,
1799       OPT_PERMS,
1800       OPT_REMOVE_SOURCE_FILES,
1801       OPT_REMOVE_SOURCE_DIRS,
1802       OPT_SCRIPT,
1803       OPT_SCRIPT_ONLY,
1804       OPT_SIZE_RANGE,
1805       OPT_USE_CACHE,
1806       OPT_USE_PGET_N,
1807       OPT_SKIP_NOACCESS,
1808       OPT_ON_CHANGE,
1809       OPT_NO_EMPTY_DIRS,
1810       OPT_DEPTH_FIRST,
1811       OPT_ASCII,
1812       OPT_SCAN_ALL_FIRST,
1813       OPT_OVERWRITE,
1814       OPT_NO_OVERWRITE,
1815       OPT_RECURSION,
1816       OPT_UPLOAD_OLDER,
1817       OPT_TRANSFER_ALL,
1818       OPT_TARGET_FLAT,
1819       OPT_DELETE_EXCLUDED,
1820    };
1821    static const struct option mirror_opts[]=
1822    {
1823       {"delete",no_argument,0,'e'},
1824       {"allow-suid",no_argument,0,'s'},
1825       {"allow-chown",no_argument,0,OPT_ALLOW_CHOWN},
1826       {"include",required_argument,0,'i'},
1827       {"exclude",required_argument,0,'x'},
1828       {"include-glob",required_argument,0,'I'},
1829       {"exclude-glob",required_argument,0,'X'},
1830       {"include-rx-from",required_argument,0,'i'+'f'},
1831       {"exclude-rx-from",required_argument,0,'x'+'f'},
1832       {"include-glob-from",required_argument,0,'I'+'f'},
1833       {"exclude-glob-from",required_argument,0,'X'+'f'},
1834       {"only-newer",no_argument,0,'n'},
1835       {"no-recursion",no_argument,0,'r'},
1836       {"no-perms",no_argument,0,'p'},
1837       {"perms",no_argument,0,OPT_PERMS},
1838       {"no-umask",no_argument,0,OPT_NO_UMASK},
1839       {"continue",no_argument,0,'c'},
1840       {"reverse",no_argument,0,'R'},
1841       {"verbose",optional_argument,0,'v'},
1842       {"newer-than",required_argument,0,'N'},
1843       {"file",required_argument,0,'f'},
1844       {"directory",required_argument,0,'F'},
1845       {"older-than",required_argument,0,OPT_OLDER_THAN},
1846       {"size-range",required_argument,0,OPT_SIZE_RANGE},
1847       {"dereference",no_argument,0,'L'},
1848       {"no-dereference",no_argument,0,OPT_NO_DEREFERENCE},
1849       {"use-cache",no_argument,0,OPT_USE_CACHE},
1850       {"Remove-source-files",no_argument,0,OPT_REMOVE_SOURCE_FILES},
1851       {"Remove-source-dirs",no_argument,0,OPT_REMOVE_SOURCE_DIRS},
1852       {"Move",no_argument,0,OPT_REMOVE_SOURCE_DIRS},
1853       {"parallel",optional_argument,0,'P'},
1854       {"ignore-time",no_argument,0,OPT_IGNORE_TIME},
1855       {"ignore-size",no_argument,0,OPT_IGNORE_SIZE},
1856       {"only-missing",no_argument,0,OPT_ONLY_MISSING},
1857       {"only-existing",no_argument,0,OPT_ONLY_EXISTING},
1858       {"log",required_argument,0,OPT_SCRIPT},
1859       {"script",    required_argument,0,OPT_SCRIPT_ONLY},
1860       {"just-print",optional_argument,0,OPT_SCRIPT_ONLY},
1861       {"dry-run",   optional_argument,0,OPT_SCRIPT_ONLY},
1862       {"delete-first",no_argument,0,OPT_DELETE_FIRST},
1863       {"use-pget-n",optional_argument,0,OPT_USE_PGET_N},
1864       {"no-symlinks",no_argument,0,OPT_NO_SYMLINKS},
1865       {"loop",no_argument,0,OPT_LOOP},
1866       {"max-errors",required_argument,0,OPT_MAX_ERRORS},
1867       {"skip-noaccess",no_argument,0,OPT_SKIP_NOACCESS},
1868       {"on-change",required_argument,0,OPT_ON_CHANGE},
1869       {"no-empty-dirs",no_argument,0,OPT_NO_EMPTY_DIRS},
1870       {"depth-first",no_argument,0,OPT_DEPTH_FIRST},
1871       {"ascii",no_argument,0,OPT_ASCII},
1872       {"target-directory",required_argument,0,'O'},
1873       {"destination-directory",required_argument,0,'O'},
1874       {"scan-all-first",no_argument,0,OPT_SCAN_ALL_FIRST},
1875       {"overwrite",no_argument,0,OPT_OVERWRITE},
1876       {"no-overwrite",no_argument,0,OPT_NO_OVERWRITE},
1877       {"recursion",required_argument,0,OPT_RECURSION},
1878       {"upload-older",no_argument,0,OPT_UPLOAD_OLDER},
1879       {"transfer-all",no_argument,0,OPT_TRANSFER_ALL},
1880       {"flat",no_argument,0,OPT_TARGET_FLAT},
1881       {"delete-excluded",no_argument,0,OPT_DELETE_EXCLUDED},
1882       {0}
1883    };
1884 
1885    int opt;
1886    unsigned flags=0;
1887    int max_error_count=0;
1888 
1889    bool use_cache=false;
1890 
1891    FileAccessRef source_session;
1892    FileAccessRef target_session;
1893 
1894    int	 verbose=0;
1895    const char *newer_than=0;
1896    const char *older_than=0;
1897    Ref<Range> size_range;
1898    bool  remove_source_files=false;
1899    bool  remove_source_dirs=false;
1900    bool	 skip_noaccess=ResMgr::QueryBool("mirror:skip-noaccess",0);
1901    int	 parallel=-1;
1902    int	 use_pget=-1;
1903    bool	 reverse=false;
1904    bool	 script_only=false;
1905    bool	 no_empty_dirs=ResMgr::QueryBool("mirror:no-empty-dirs",0);
1906    const char *script_file=0;
1907    const char *on_change=0;
1908    const char *recursion_mode=0;
1909    bool single_file=false;
1910    bool single_dir=false;
1911 
1912    Ref<PatternSet> exclude;
1913    Ref<PatternSet> top_exclude;
1914 
1915    if(!ResMgr::QueryBool("mirror:set-permissions",0))
1916       flags|=MirrorJob::NO_PERMS;
1917    if(ResMgr::QueryBool("mirror:dereference",0))
1918       flags|=MirrorJob::RETR_SYMLINKS;
1919    if(ResMgr::QueryBool("mirror:overwrite",0))
1920       flags|=MirrorJob::OVERWRITE;
1921 
1922    const char *source_dir=NULL;
1923    const char *target_dir=NULL;
1924 
1925    args->rewind();
1926    while((opt=args->getopt_long("esi:x:I:X:nrpcRvN:LP:af:F:O:",mirror_opts,0))!=EOF)
1927    {
1928       switch(opt)
1929       {
1930       case('e'):
1931 	 flags|=MirrorJob::DELETE;
1932 	 break;
1933       case('s'):
1934 	 flags|=MirrorJob::ALLOW_SUID;
1935 	 break;
1936       case(OPT_ALLOW_CHOWN):
1937 	 flags|=MirrorJob::ALLOW_CHOWN;
1938 	 break;
1939       case('a'):
1940 	 flags|=MirrorJob::ALLOW_SUID|MirrorJob::ALLOW_CHOWN|MirrorJob::NO_UMASK;
1941 	 break;
1942       case('r'):
1943 	 recursion_mode="never";
1944 	 break;
1945       case('n'):
1946 	 flags|=MirrorJob::ONLY_NEWER;
1947 	 break;
1948       case('p'):
1949 	 flags|=MirrorJob::NO_PERMS;
1950 	 break;
1951       case(OPT_PERMS):
1952 	 flags&=~MirrorJob::NO_PERMS;
1953 	 break;
1954       case('c'):
1955 	 flags|=MirrorJob::CONTINUE;
1956 	 break;
1957       case('x'):
1958       case('i'):
1959       case('X'):
1960       case('I'):
1961       {
1962 	 const char *err=MirrorJob::AddPattern(exclude,opt,optarg);
1963 	 if(err)
1964 	 {
1965 	    eprintf("%s: %s\n",args->a0(),err);
1966 	    goto no_job;
1967 	 }
1968 	 break;
1969       }
1970       case('x'+'f'):
1971       case('i'+'f'):
1972       case('X'+'f'):
1973       case('I'+'f'):
1974       {
1975 	 const char *err=MirrorJob::AddPatternsFrom(exclude,opt-'f',optarg);
1976 	 if(err)
1977 	 {
1978 	    eprintf("%s: %s\n",args->a0(),err);
1979 	    goto no_job;
1980 	 }
1981 	 break;
1982       }
1983       case('R'):
1984 	 reverse=true;
1985 	 break;
1986       case('L'):
1987 	 flags|=MirrorJob::RETR_SYMLINKS;
1988 	 break;
1989       case(OPT_NO_DEREFERENCE):
1990 	 flags&=~MirrorJob::RETR_SYMLINKS;
1991 	 break;
1992       case('v'):
1993 	 if(optarg)
1994 	    verbose=atoi(optarg);
1995 	 else
1996 	    verbose++;
1997 	 if(verbose>1)
1998 	    flags|=MirrorJob::REPORT_NOT_DELETED;
1999 	 break;
2000       case('N'):
2001 	 newer_than=optarg;
2002 	 break;
2003       case('f'): // mirror for a single file (or glob pattern).
2004 	 single_file=true;
2005 	 /*fallthrough*/
2006       case('F'): // mirror for a single directory (or glob pattern).
2007       {
2008 	 xstring pattern(basename_ptr(optarg));
2009 	 if(opt=='F') {
2010 	    single_dir=true;
2011 	    if(pattern.last_char()!='/')
2012 	       pattern.append('/');
2013 	 }
2014 	 if(!top_exclude)
2015 	    top_exclude=new PatternSet();
2016 	 top_exclude->Add(PatternSet::INCLUDE,new PatternSet::Glob(pattern));
2017 	 const char *dir=dirname(optarg);
2018 	 if(source_dir && strcmp(source_dir,dir)) {
2019 	    eprintf(_("%s: multiple --file or --directory options must have the same base directory\n"),args->a0());
2020 	    goto no_job;
2021 	 }
2022 	 if(!source_dir)
2023 	    source_dir=alloca_strdup(dir); // save the temp string
2024 	 break;
2025       }
2026       case('O'):
2027 	 target_dir=optarg;
2028 	 break;
2029       case(OPT_OLDER_THAN):
2030 	 older_than=optarg;
2031 	 break;
2032       case(OPT_SIZE_RANGE):
2033 	 size_range=new Range(optarg);
2034 	 if(size_range->Error())
2035 	 {
2036 	    eprintf("%s: --size-range \"%s\": %s\n",
2037 	       args->a0(),optarg,size_range->ErrorText());
2038 	    goto no_job;
2039 	 }
2040 	 break;
2041       case(OPT_NO_UMASK):
2042 	 flags|=MirrorJob::NO_UMASK;
2043 	 break;
2044       case(OPT_USE_CACHE):
2045 	 use_cache=true;
2046 	 break;
2047       case(OPT_REMOVE_SOURCE_FILES):
2048 	 remove_source_files=true;
2049 	 break;
2050       case(OPT_REMOVE_SOURCE_DIRS):
2051 	 remove_source_dirs=true;
2052 	 break;
2053       case(OPT_IGNORE_TIME):
2054 	 flags|=MirrorJob::IGNORE_TIME;
2055 	 break;
2056       case(OPT_IGNORE_SIZE):
2057 	 flags|=MirrorJob::IGNORE_SIZE;
2058 	 break;
2059       case(OPT_ONLY_MISSING):
2060 	 flags|=MirrorJob::IGNORE_TIME|MirrorJob::IGNORE_SIZE;
2061 	 break;
2062       case('P'):
2063 	 if(optarg)
2064 	    parallel=atoi(optarg);
2065 	 else
2066 	    parallel=3;
2067 	 break;
2068       case(OPT_USE_PGET_N):
2069 	 if(optarg)
2070 	    use_pget=atoi(optarg);
2071 	 else
2072 	    use_pget=3;
2073 	 break;
2074       case(OPT_SCRIPT_ONLY):
2075 	 script_only=true;
2076       case(OPT_SCRIPT):
2077 	 script_file=optarg;
2078 	 if(script_file==0)
2079 	    script_file="-";
2080 	 break;
2081       case(OPT_DELETE_FIRST):
2082 	 flags|=MirrorJob::REMOVE_FIRST|MirrorJob::DELETE;
2083 	 break;
2084       case(OPT_NO_SYMLINKS):
2085 	 flags|=MirrorJob::NO_SYMLINKS;
2086 	 break;
2087       case(OPT_LOOP):
2088 	 flags|=MirrorJob::LOOP;
2089 	 break;
2090       case(OPT_MAX_ERRORS):
2091 	 max_error_count=atoi(optarg);
2092 	 break;
2093       case(OPT_SKIP_NOACCESS):
2094 	 skip_noaccess=true;
2095 	 break;
2096       case(OPT_ON_CHANGE):
2097 	 on_change=optarg;
2098 	 break;
2099       case(OPT_ONLY_EXISTING):
2100 	 flags|=MirrorJob::ONLY_EXISTING;
2101 	 break;
2102       case(OPT_NO_EMPTY_DIRS):
2103 	 no_empty_dirs=true;
2104 	 break;
2105       case(OPT_DEPTH_FIRST):
2106 	 flags|=MirrorJob::DEPTH_FIRST;
2107 	 break;
2108       case(OPT_SCAN_ALL_FIRST):
2109 	 flags|=MirrorJob::SCAN_ALL_FIRST|MirrorJob::DEPTH_FIRST;
2110 	 break;
2111       case(OPT_ASCII):
2112 	 flags|=MirrorJob::ASCII|MirrorJob::IGNORE_SIZE;
2113 	 break;
2114       case(OPT_OVERWRITE):
2115 	 flags|=MirrorJob::OVERWRITE;
2116 	 break;
2117       case(OPT_NO_OVERWRITE):
2118 	 flags&=~MirrorJob::OVERWRITE;
2119 	 break;
2120       case(OPT_RECURSION):
2121 	 recursion_mode=optarg;
2122 	 break;
2123       case(OPT_UPLOAD_OLDER):
2124 	 flags|=MirrorJob::UPLOAD_OLDER;
2125 	 break;
2126       case(OPT_TRANSFER_ALL):
2127 	 flags|=MirrorJob::TRANSFER_ALL;
2128 	 break;
2129       case(OPT_TARGET_FLAT):
2130 	 flags|=MirrorJob::TARGET_FLAT|MirrorJob::SCAN_ALL_FIRST|MirrorJob::DEPTH_FIRST;
2131 	 break;
2132       case(OPT_DELETE_EXCLUDED):
2133 	 flags|=MirrorJob::DELETE_EXCLUDED;
2134 	 break;
2135       case('?'):
2136 	 eprintf(_("Try `help %s' for more information.\n"),args->a0());
2137       no_job:
2138 	 return 0;
2139       }
2140    }
2141 
2142    if(exclude && xstrcasecmp(recursion_mode,"never"))
2143    {
2144       /* Users usually don't want to exclude all directories when recursing */
2145       if(exclude->GetFirstType()==PatternSet::INCLUDE)
2146 	 exclude->AddFirst(PatternSet::INCLUDE,new PatternSet::Regex("/$"));
2147    }
2148 
2149    /* add default exclusion if no explicit patterns were specified */
2150    if(!exclude)
2151       MirrorJob::AddPattern(exclude,'\0',0);
2152 
2153    args->back();
2154 
2155    const char *arg=args->getnext();
2156    if(arg)
2157    {
2158       if(source_dir)
2159       {
2160 	 eprintf(_("%s: ambiguous source directory (`%s' or `%s'?)\n"),args->a0(),
2161 	    source_dir,arg);
2162 	 goto no_job;
2163       }
2164       source_dir=arg;
2165       ParsedURL source_url(source_dir);
2166       if(source_url.proto && source_url.path)
2167       {
2168 	 source_session=FileAccess::New(&source_url);
2169 	 if(!source_session)
2170 	 {
2171 	    eprintf("%s: %s%s\n",args->a0(),source_url.proto.get(),
2172 		     _(" - not supported protocol"));
2173 	    goto no_job;
2174 	 }
2175 	 source_dir=alloca_strdup(source_url.path);
2176       }
2177       arg=args->getnext();
2178       if(arg)
2179       {
2180 	 if(target_dir)
2181 	 {
2182 	    eprintf(_("%s: ambiguous target directory (`%s' or `%s'?)\n"),args->a0(),
2183 	       target_dir,arg);
2184 	    goto no_job;
2185 	 }
2186 	 target_dir=arg;
2187 	 ParsedURL target_url(target_dir);
2188 	 if(target_url.proto && target_url.path)
2189 	 {
2190 	    target_session=FileAccess::New(&target_url);
2191 	    if(!target_session)
2192 	    {
2193 	       eprintf("%s: %s%s\n",args->a0(),target_url.proto.get(),
2194 			_(" - not supported protocol"));
2195 	       goto no_job;
2196 	    }
2197 	    target_dir=alloca_strdup(target_url.path);
2198 	 }
2199 	 if(last_char(arg)=='/' && basename_ptr(arg)[0]!='/' && last_char(source_dir)!='/')
2200 	 {
2201 	    // user wants source dir name appended.
2202 	    const char *base=basename_ptr(source_dir);
2203 	    if(base[0]!='/' && strcmp(base,basename_ptr(arg))) {
2204 	       target_dir=xstring::cat(target_dir,base,NULL);
2205 	       target_dir=alloca_strdup(target_dir); // save the buffer
2206 	    }
2207 	 }
2208       }
2209       else
2210       {
2211 	 target_dir=basename_ptr(source_dir);
2212 	 if(target_dir[0]=='/')
2213 	    target_dir=".";
2214 	 else if(target_dir[0]=='~') {
2215 	    target_dir=dir_file(".",target_dir);
2216 	    target_dir=alloca_strdup(target_dir); // save the buffer
2217 	 }
2218       }
2219    }
2220 
2221    if(!source_dir) {
2222       if(ResMgr::QueryBool("mirror:require-source",0)) {
2223 	 eprintf(_("%s: source directory is required (mirror:require-source is set)\n"),args->a0());
2224 	 return 0;
2225       }
2226       source_dir=".";
2227    }
2228    if(!target_dir)
2229       target_dir=".";
2230 
2231    if(!reverse)
2232    {
2233       if(!source_session)
2234 	 source_session=parent->session->Clone();
2235       if(!target_session)
2236 	 target_session=FileAccess::New("file");
2237    }
2238    else //reverse
2239    {
2240       if(!source_session)
2241 	 source_session=FileAccess::New("file");
2242       if(!target_session)
2243 	 target_session=parent->session->Clone();
2244       flags|=MirrorJob::REVERSE;
2245    }
2246 
2247    if(no_empty_dirs)
2248       flags|=MirrorJob::NO_EMPTY_DIRS|MirrorJob::DEPTH_FIRST;
2249 
2250    if(recursion_mode && strcasecmp(recursion_mode,"always")
2251    && (flags&MirrorJob::DEPTH_FIRST)) {
2252       eprintf("%s: --recursion-mode=%s conflicts with other specified options\n",
2253 	 args->a0(),recursion_mode);
2254       return 0;
2255    }
2256 
2257    if(parallel<0) {
2258       int parallel1=ResMgr::Query("mirror:parallel-transfer-count",source_session->GetHostName());
2259       int parallel2=ResMgr::Query("mirror:parallel-transfer-count",target_session->GetHostName());
2260       if(parallel1>0)
2261 	 parallel=parallel1;
2262       if(parallel2>0 && (parallel<0 || parallel>parallel2))
2263 	 parallel=parallel2;
2264    }
2265    if(use_pget<0) {
2266       int use_pget1=ResMgr::Query("mirror:use-pget-n",source_session->GetHostName());
2267       int use_pget2=ResMgr::Query("mirror:use-pget-n",target_session->GetHostName());
2268       if(use_pget1>0)
2269 	 use_pget=use_pget1;
2270       if(use_pget2>0 && (use_pget<0 || use_pget>use_pget2))
2271 	 use_pget=use_pget2;
2272    }
2273 
2274    JobRef<MirrorJob> j(new MirrorJob(0,source_session.borrow(),target_session.borrow(),source_dir,target_dir));
2275    j->SetFlags(flags,1);
2276    j->SetVerbose(verbose);
2277    j->SetExclude(exclude.borrow());
2278    j->SetTopExclude(top_exclude.borrow());
2279 
2280    if(newer_than)
2281       j->SetNewerThan(newer_than);
2282    if(older_than)
2283       j->SetOlderThan(older_than);
2284    if(size_range)
2285       j->SetSizeRange(size_range.borrow());
2286    j->UseCache(use_cache);
2287    if(remove_source_files)
2288       j->RemoveSourceFiles();
2289    if(remove_source_dirs)
2290       j->RemoveSourceDirs();
2291    if(skip_noaccess)
2292       j->SkipNoAccess();
2293    if(parallel<0)
2294       parallel=0;
2295    if(parallel>64)
2296       parallel=64;   // a (in)sane limit.
2297    if(parallel)
2298       j->SetParallel(parallel);
2299    if(use_pget>1 && !(flags&MirrorJob::ASCII))
2300       j->SetPGet(use_pget);
2301 
2302    if(!recursion_mode && single_file && !single_dir)
2303       recursion_mode="never";
2304 
2305    if(recursion_mode) {
2306       const char *err=j->SetRecursionMode(recursion_mode);
2307       if(err) {
2308 	 eprintf("%s: %s\n",args->a0(),err);
2309 	 return 0;
2310       }
2311    }
2312    if(script_file)
2313    {
2314       const char *err=j->SetScriptFile(script_file);
2315       if(err)
2316       {
2317 	 eprintf("%s: %s\n",args->a0(),err);
2318 	 return 0;
2319       }
2320    }
2321    if(script_only)
2322    {
2323       j->ScriptOnly();
2324       if(!script_file)
2325 	 j->SetScriptFile("-");
2326    }
2327    j->SetMaxErrorCount(max_error_count);
2328    if(on_change)
2329       j->SetOnChange(on_change);
2330 
2331    return j.borrow();
2332 
2333 #undef args
2334 }
2335 
2336 #include "modconfig.h"
2337 #ifndef MODULE_CMD_MIRROR
2338 # define module_init cmd_mirror_module_init
2339 #endif
module_init()2340 CDECL void module_init()
2341 {
2342    CmdExec::RegisterCommand("mirror",cmd_mirror,0,
2343       N_("\n"
2344 	 "Mirror specified remote directory to local directory\n"
2345 	 "\n"
2346 	 " -R, --reverse          reverse mirror (put files)\n"
2347 	 "Lots of other options are documented in the man page lftp(1).\n"
2348 	 "\n"
2349 	 "When using -R, the first directory is local and the second is remote.\n"
2350 	 "If the second directory is omitted, basename of the first directory is used.\n"
2351 	 "If both directories are omitted, current local and remote directories are used.\n"
2352 	 "\n"
2353 	 "See the man page lftp(1) for a complete documentation.\n"
2354       )
2355    );
2356 }
2357