1 /*
2 * lftp - file transfer program
3 *
4 * Copyright (c) 1996-2017 by Alexander V. Lukyanov (lav@yars.free.net)
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include <config.h>
21
22 #include <sys/types.h>
23 #include <sys/stat.h>
24 #include <errno.h>
25 #include <fcntl.h>
26 #include <unistd.h>
27 #include <assert.h>
28 #include <mbswidth.h>
29 #include "MirrorJob.h"
30 #include "CmdExec.h"
31 #include "rmJob.h"
32 #include "mvJob.h"
33 #include "ChmodJob.h"
34 #include "mkdirJob.h"
35 #include "misc.h"
36 #include "plural.h"
37 #include "FindJob.h"
38 #include "url.h"
39 #include "CopyJob.h"
40 #include "pgetJob.h"
41 #include "log.h"
42
43 #define set_state(s) do { state=(s); \
44 Log::global->Format(11,"mirror(%p) enters state %s\n", this, #s); } while(0)
45 #define waiting_num waiting.count()
46 #define transfer_count root_mirror->root_transfer_count
47
FormatStatus(xstring & s,int v,const char * tab)48 xstring& MirrorJob::FormatStatus(xstring& s,int v,const char *tab)
49 {
50 if(Done())
51 goto final;
52
53 switch(state)
54 {
55 case(INITIAL_STATE):
56 case(FINISHING):
57 case(DONE):
58 case(WAITING_FOR_TRANSFER):
59 case(TARGET_REMOVE_OLD):
60 case(TARGET_REMOVE_OLD_FIRST):
61 case(TARGET_CHMOD):
62 case(TARGET_MKDIR):
63 case(SOURCE_REMOVING_SAME):
64 case(LAST_EXEC):
65 break;
66
67 case(MAKE_TARGET_DIR):
68 s.appendf("\tmkdir `%s' [%s]\n",target_dir.get(),target_session->CurrentStatus());
69 break;
70
71 case(CHANGING_DIR_SOURCE):
72 case(CHANGING_DIR_TARGET):
73 if(target_session->IsOpen())
74 s.appendf("\tcd `%s' [%s]\n",target_dir.get(),target_session->CurrentStatus());
75 if(source_session->IsOpen())
76 s.appendf("\tcd `%s' [%s]\n",source_dir.get(),source_session->CurrentStatus());
77 break;
78
79 case(GETTING_LIST_INFO):
80 if(target_list_info)
81 {
82 if(target_relative_dir)
83 s.appendf("\t%s: %s\n",target_relative_dir.get(),target_list_info->Status());
84 else
85 s.appendf("\t%s\n",target_list_info->Status());
86 }
87 if(source_list_info)
88 {
89 if(source_relative_dir)
90 s.appendf("\t%s: %s\n",source_relative_dir.get(),source_list_info->Status());
91 else
92 s.appendf("\t%s\n",source_list_info->Status());
93 }
94 break;
95 }
96 return s;
97
98 final:
99 if(stats.dirs>0)
100 s.appendf(plural("%sTotal: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n",
101 stats.dirs,stats.tot_files,stats.tot_symlinks),
102 tab,stats.dirs,stats.tot_files,stats.tot_symlinks);
103 if(stats.new_files || stats.new_symlinks)
104 s.appendf(plural("%sNew: %d file$|s$, %d symlink$|s$\n",
105 stats.new_files,stats.new_symlinks),
106 tab,stats.new_files,stats.new_symlinks);
107 if(stats.mod_files || stats.mod_symlinks)
108 s.appendf(plural("%sModified: %d file$|s$, %d symlink$|s$\n",
109 stats.mod_files,stats.mod_symlinks),
110 tab,stats.mod_files,stats.mod_symlinks);
111 if(stats.bytes)
112 s.appendf("%s%s\n",tab,CopyJob::FormatBytesTimeRate(stats.bytes,transfer_time_elapsed));
113 if(stats.del_dirs || stats.del_files || stats.del_symlinks)
114 s.appendf(plural(FlagSet(DELETE) ?
115 "%sRemoved: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n"
116 :"%sTo be removed: %d director$y|ies$, %d file$|s$, %d symlink$|s$\n",
117 stats.del_dirs,stats.del_files,stats.del_symlinks),
118 tab,stats.del_dirs,stats.del_files,stats.del_symlinks);
119 if(stats.error_count)
120 s.appendf(plural("%s%d error$|s$ detected\n",stats.error_count),
121 tab,stats.error_count);
122 return s;
123 }
124
ShowRunStatus(const SMTaskRef<StatusLine> & s)125 void MirrorJob::ShowRunStatus(const SMTaskRef<StatusLine>& s)
126 {
127 int w=s->GetWidthDelayed();
128 switch(state)
129 {
130 case(INITIAL_STATE):
131 break;
132
133 // these have a sub-job
134 case(WAITING_FOR_TRANSFER):
135 case(TARGET_REMOVE_OLD):
136 case(TARGET_REMOVE_OLD_FIRST):
137 case(TARGET_CHMOD):
138 case(TARGET_MKDIR):
139 case(SOURCE_REMOVING_SAME):
140 case(FINISHING):
141 case(DONE):
142 case(LAST_EXEC):
143 Job::ShowRunStatus(s);
144 break;
145
146 case(MAKE_TARGET_DIR):
147 s->Show("mkdir `%s' [%s]",target_dir.get(),target_session->CurrentStatus());
148 break;
149
150 case(CHANGING_DIR_SOURCE):
151 case(CHANGING_DIR_TARGET):
152 if(target_session->IsOpen() && (!source_session->IsOpen() || now%4>=2))
153 s->Show("cd `%s' [%s]",target_dir.get(),target_session->CurrentStatus());
154 else if(source_session->IsOpen())
155 s->Show("cd `%s' [%s]",source_dir.get(),source_session->CurrentStatus());
156 break;
157
158 case(GETTING_LIST_INFO):
159 if(target_list_info && (!source_list_info || now%4>=2))
160 {
161 const char *status=target_list_info->Status();
162 int status_w=mbswidth(status, 0);
163 int dw=w-status_w;
164 if(dw<20)
165 dw=20;
166 if(target_relative_dir)
167 s->Show("%s: %s",squeeze_file_name(target_relative_dir,dw),status);
168 else
169 s->Show("%s",status);
170 }
171 else if(source_list_info)
172 {
173 const char *status=source_list_info->Status();
174 int status_w=mbswidth(status, 0);
175 int dw=w-status_w;
176 if(dw<20)
177 dw=20;
178 if(source_relative_dir)
179 s->Show("%s: %s",squeeze_file_name(source_relative_dir,dw),status);
180 else
181 s->Show("%s",status);
182 }
183 break;
184 }
185 }
186
FormatShortStatus(xstring & s)187 xstring& MirrorJob::FormatShortStatus(xstring& s)
188 {
189 if(bytes_to_transfer>0 && (!parent_mirror || parent_mirror->bytes_to_transfer!=bytes_to_transfer)) {
190 long long curr_bytes_transferred=GetBytesCount();
191 if(parent_mirror)
192 curr_bytes_transferred+=bytes_transferred;
193 s.appendf("%s/%s (%d%%)",
194 xhuman(curr_bytes_transferred),xhuman(bytes_to_transfer),
195 percent(curr_bytes_transferred,bytes_to_transfer));
196 double rate=GetTransferRate();
197 if(rate>=1)
198 s.append(' ').append(Speedometer::GetStrProper(rate));
199 }
200 return s;
201 }
202
TransferStarted(CopyJob * cp)203 void MirrorJob::TransferStarted(CopyJob *cp)
204 {
205 if(transfer_count==0)
206 root_mirror->transfer_start_ts=now;
207 JobStarted(cp);
208 }
JobStarted(Job * j)209 void MirrorJob::JobStarted(Job *j)
210 {
211 AddWaiting(j);
212 transfer_count++;
213 }
TransferFinished(Job * j)214 void MirrorJob::TransferFinished(Job *j)
215 {
216 long long bytes_count=j->GetBytesCount();
217 AddBytesTransferred(bytes_count);
218 stats.bytes+=bytes_count;
219 stats.time +=j->GetTimeSpent();
220 if(j->ExitCode()==0 && verbose_report>=2) {
221 xstring finished;
222 const xstring& cmd=j->GetCmdLine();
223 if(cmd[0]=='\\')
224 finished.append(cmd+1,cmd.length()-1);
225 else
226 finished.append(cmd);
227 const xstring& rate=Speedometer::GetStrProper(j->GetTransferRate());
228 if(rate.length()>0)
229 finished.append(" (").append(rate).append(')');
230 if(!(FlagSet(SCAN_ALL_FIRST) && finished.begins_with("mirror")))
231 Report(_("Finished %s"),finished.get());
232 }
233 JobFinished(j);
234 if(transfer_count==0)
235 root_mirror->transfer_time_elapsed += now-root_mirror->transfer_start_ts;
236 }
JobFinished(Job * j)237 void MirrorJob::JobFinished(Job *j)
238 {
239 if(j->ExitCode()!=0)
240 stats.error_count++;
241 RemoveWaiting(j);
242 Delete(j);
243 assert(transfer_count>0);
244 transfer_count--;
245 }
246
GetBytesCount()247 off_t MirrorJob::GetBytesCount()
248 {
249 long long bytes_count=Job::GetBytesCount();
250 if(!parent_mirror) {
251 // bytes_transferred is cumulative over the mirror tree,
252 // add it on the top only
253 bytes_count+=bytes_transferred;
254 }
255 return bytes_count;
256 }
GetTimeSpent()257 double MirrorJob::GetTimeSpent()
258 {
259 double t=transfer_time_elapsed;
260 if(transfer_count>0)
261 t+=now-root_mirror->transfer_start_ts;
262 return t;
263 }
264
HandleFile(FileInfo * file)265 void MirrorJob::HandleFile(FileInfo *file)
266 {
267 int res;
268 struct stat st;
269
270 // TODO: get rid of local hacks.
271
272 const char *dst_name=file->name;
273 if(FlagSet(TARGET_FLAT))
274 dst_name=basename_ptr(dst_name);
275
276 // dir_name returns pointer to static data - need to dup it.
277 const char *source_name=dir_file(source_dir,file->name);
278 source_name=alloca_strdup(source_name);
279 const char *target_name=dir_file(target_dir,dst_name);
280 target_name=alloca_strdup(target_name);
281
282 const char *source_name_rel=dir_file(source_relative_dir,file->name);
283 source_name_rel=alloca_strdup(source_name_rel);
284 const char *target_name_rel=dir_file(target_relative_dir,dst_name);
285 target_name_rel=alloca_strdup(target_name_rel);
286
287 FileInfo::type filetype=FileInfo::NORMAL;
288 if(file->Has(file->TYPE))
289 filetype=file->filetype;
290 else
291 {
292 FileInfo *target=target_set->FindByName(file->name);
293 if(target && target->Has(target->TYPE))
294 filetype=target->filetype;
295 }
296
297 switch(filetype)
298 {
299 case(FileInfo::NORMAL):
300 case(FileInfo::REDIRECT):
301 {
302 bool remove_target=false;
303 bool cont_this=false;
304 bool use_pget=(pget_n>1) && target_is_local;
305 if(file->Has(file->SIZE) && file->size<pget_minchunk*2)
306 use_pget=false;
307 if(target_is_local)
308 {
309 if(lstat(target_name,&st)!=-1)
310 {
311 // few safety checks.
312 FileInfo *old=new_files_set->FindByName(file->name);
313 if(old)
314 goto skip; // file has appeared after mirror start
315 old=old_files_set->FindByName(file->name);
316 if(old && ((old->Has(old->SIZE) && old->size!=st.st_size)
317 ||(old->Has(old->DATE) && old->date!=st.st_mtime)))
318 goto skip; // the file has changed after mirror start
319 if(!script_only && access(target_name,W_OK)==-1)
320 {
321 // try to enable write access.
322 chmod(target_name,st.st_mode|0200);
323 }
324 }
325 }
326 FileInfo *old=target_set->FindByName(FileCopy::TempFileName(file->name));
327 if(old)
328 {
329 if(FlagSet(CONTINUE)
330 && old->Has(file->TYPE) && old->filetype==old->NORMAL
331 && (FlagSet(IGNORE_TIME) ||
332 (file->Has(file->DATE) && old->Has(old->DATE)
333 && file->date + file->date.ts_prec < old->date - old->date.ts_prec))
334 && file->Has(file->SIZE) && old->Has(old->SIZE)
335 && file->size >= old->size)
336 {
337 cont_this=true;
338 stats.mod_files++;
339 }
340 else if(!to_rm_mismatched->FindByName(file->name))
341 {
342 if(!FlagSet(OVERWRITE)) {
343 remove_target=true;
344 Report(_("Removing old file `%s'"),target_name_rel);
345 } else {
346 Report(_("Overwriting old file `%s'"),target_name_rel);
347 }
348 stats.mod_files++;
349 }
350 else
351 stats.new_files++;
352 }
353 else if(FlagSet(ONLY_EXISTING))
354 {
355 Report(_("Skipping file `%s' (only-existing)"),source_name_rel);
356 goto skip;
357 }
358 else
359 stats.new_files++;
360
361 Report(_("Transferring file `%s'"),source_name_rel);
362
363 if(script)
364 {
365 ArgV args(use_pget?"pget":"get");
366 if(use_pget)
367 {
368 args.Append("-n");
369 args.Append(pget_n);
370 }
371 if(cont_this)
372 args.Append("-c");
373 if(remove_target)
374 args.Append("-e");
375 if(FlagSet(ASCII))
376 args.Append("-a");
377 if(remove_source_files)
378 args.Append("-E");
379 args.Append("-O");
380 args.Append(target_is_local?target_dir.get()
381 :target_session->GetConnectURL().get());
382 args.Append(source_session->GetFileURL(file->name));
383 xstring_ca cmd(args.CombineQuoted());
384 fprintf(script,"%s\n",cmd.get());
385 if(script_only)
386 goto skip;
387 }
388
389 FileCopyPeer *src_peer=0;
390 if(source_is_local)
391 src_peer=new FileCopyPeerFDStream(new FileStream(source_name,O_RDONLY),FileCopyPeer::GET);
392 else
393 src_peer=new FileCopyPeerFA(source_session->Clone(),file->name,FA::RETRIEVE);
394
395 FileCopyPeer *dst_peer=0;
396 if(target_is_local)
397 dst_peer=new FileCopyPeerFDStream(new FileStream(target_name,O_WRONLY|O_CREAT|(cont_this?0:O_TRUNC)),FileCopyPeer::PUT);
398 else
399 dst_peer=new FileCopyPeerFA(target_session->Clone(),dst_name,FA::STORE);
400
401 FileCopy *c=FileCopy::New(src_peer,dst_peer,cont_this);
402 if(remove_source_files)
403 c->RemoveSourceLater();
404 if(remove_target)
405 c->RemoveTargetFirst();
406 if(FlagSet(ASCII))
407 c->Ascii();
408 CopyJob *cp=(use_pget ? new pgetJob(c,file->name,pget_n) : new CopyJob(c,file->name,"mirror"));
409 if(file->Has(file->DATE))
410 cp->SetDate(file->date);
411 if(file->Has(file->SIZE) && !FlagSet(IGNORE_SIZE))
412 cp->SetSize(file->size);
413 TransferStarted(cp);
414 cp->cmdline.vset("\\transfer `",source_name_rel,"'",NULL);
415
416 set_state(WAITING_FOR_TRANSFER);
417 break;
418 }
419 case(FileInfo::DIRECTORY):
420 {
421 if(recursion_mode==RECURSION_NEVER || FlagSet(NO_RECURSION))
422 goto skip;
423
424 bool create_target_subdir=true;
425 const FileInfo *old=0;
426
427 if(FlagSet(TARGET_FLAT)) {
428 create_target_subdir=false;
429 target_name=target_dir;
430 goto do_submirror;
431 }
432
433 if(target_set)
434 old=target_set->FindByName(file->name);
435 if(!old)
436 {
437 if(FlagSet(ONLY_EXISTING))
438 {
439 Report(_("Skipping directory `%s' (only-existing)"),target_name_rel);
440 goto skip;
441 }
442 }
443 else if(old->TypeIs(old->DIRECTORY))
444 {
445 create_target_subdir=false;
446 }
447 if(target_is_local && !script_only)
448 {
449 if((FlagSet(RETR_SYMLINKS)?stat:lstat)(target_name,&st)!=-1)
450 {
451 if(S_ISDIR(st.st_mode))
452 {
453 // try to enable write access
454 // only if not enabled as chmod can clear sgid flags on directories
455 if(st.st_mode!=(st.st_mode|0700))
456 chmod(target_name,st.st_mode|0700);
457 create_target_subdir=false;
458 }
459 else
460 {
461 Report(_("Removing old local file `%s'"),target_name_rel);
462 if(remove(target_name)==-1)
463 {
464 eprintf("mirror: remove(%s): %s\n",target_name,strerror(errno));
465 goto skip;
466 }
467 create_target_subdir=true;
468 }
469 }
470 }
471
472 do_submirror:
473 // launch sub-mirror
474 MirrorJob *mj=new MirrorJob(this,
475 source_session->Clone(),target_session->Clone(),
476 source_name,target_name);
477 AddWaiting(mj);
478 mj->cmdline.vset("\\mirror `",source_name_rel,"'",NULL);
479
480 mj->source_relative_dir.set(source_name_rel);
481 mj->target_relative_dir.set(target_name_rel);
482
483 mj->create_target_dir=create_target_subdir;
484
485 if(verbose_report>=3) {
486 if(FlagSet(SCAN_ALL_FIRST))
487 Report(_("Scanning directory `%s'"),mj->target_relative_dir.get());
488 else
489 Report(_("Mirroring directory `%s'"),mj->target_relative_dir.get());
490 }
491
492 break;
493 }
494 case(FileInfo::SYMLINK):
495 {
496 if(FlagSet(NO_SYMLINKS))
497 goto skip;
498
499 if(!file->symlink)
500 goto skip;
501
502 if(!target_is_local)
503 {
504 if(script)
505 {
506 ArgV args("ln");
507 args.Append("-s");
508 args.Append(file->symlink);
509 args.Append(target_name);
510 xstring_ca cmd(args.CombineQuoted());
511 fprintf(script,"%s\n",cmd.get());
512 if(script_only)
513 goto skip;
514 }
515 bool remove_target=false;
516 FileInfo *old=target_set->FindByName(file->name);
517 if(old && !to_rm_mismatched->FindByName(file->name))
518 {
519 Report(_("Removing old file `%s'"),target_name_rel);
520 remove_target=true;
521 stats.mod_symlinks++;
522 }
523 else
524 stats.new_symlinks++;
525 Report(_("Making symbolic link `%s' to `%s'"),target_name_rel,file->symlink.get());
526 mvJob *j=new mvJob(target_session->Clone(),file->symlink,target_name,FA::SYMLINK);
527 if(remove_target)
528 j->RemoveTargetFirst();
529 JobStarted(j);
530 RemoveSourceLater(file);
531 break;
532 }
533
534 if(script)
535 {
536 ArgV args("shell");
537 args.Append("ln");
538 args.Append("-sf");
539 args.Append(shell_encode(file->symlink));
540 args.Append(shell_encode(target_name));
541 xstring_ca cmd(args.CombineQuoted());
542 fprintf(script,"%s\n",cmd.get());
543 if(script_only)
544 goto skip;
545 }
546
547 struct stat st;
548 if(lstat(target_name,&st)!=-1)
549 {
550 Report(_("Removing old local file `%s'"),target_name_rel);
551 stats.mod_symlinks++;
552 if(remove(target_name)==-1)
553 {
554 eprintf("mirror: remove(%s): %s\n",target_name,strerror(errno));
555 goto skip;
556 }
557 }
558 else
559 {
560 if(FlagSet(ONLY_EXISTING))
561 {
562 Report(_("Skipping symlink `%s' (only-existing)"),target_name_rel);
563 goto skip;
564 }
565 stats.new_symlinks++;
566 }
567 Report(_("Making symbolic link `%s' to `%s'"),target_name_rel,file->symlink.get());
568 res=symlink(file->symlink,target_name);
569 if(res==-1)
570 eprintf("mirror: symlink(%s): %s\n",target_name,strerror(errno));
571 RemoveSourceLater(file);
572 break;
573 }
574 case FileInfo::UNKNOWN:
575 break;
576 }
577 skip:
578 return;
579 }
580
InitSets()581 void MirrorJob::InitSets()
582 {
583 if(FlagSet(TARGET_FLAT) && !parent_mirror && target_set)
584 source_set->Sort(FileSet::BYNAME_FLAT);
585
586 source_set->Count(NULL,&stats.tot_files,&stats.tot_symlinks,&stats.tot_files);
587
588 to_rm=new FileSet(target_set);
589 to_rm->SubtractAny(source_set);
590
591 if(FlagSet(DELETE_EXCLUDED) && target_set_excluded)
592 to_rm->Merge(target_set_excluded);
593
594 to_transfer=new FileSet(source_set);
595
596 if(!FlagSet(TRANSFER_ALL)) {
597 same=new FileSet(source_set);
598
599 int ignore=0;
600 if(FlagSet(ONLY_NEWER))
601 ignore|=FileInfo::IGNORE_SIZE_IF_OLDER|FileInfo::IGNORE_DATE_IF_OLDER;
602 if(!FlagSet(UPLOAD_OLDER) && strcmp(target_session->GetProto(),"file"))
603 ignore|=FileInfo::IGNORE_DATE_IF_OLDER;
604 if(FlagSet(IGNORE_TIME))
605 ignore|=FileInfo::DATE;
606 if(FlagSet(IGNORE_SIZE))
607 ignore|=FileInfo::SIZE;
608 to_transfer->SubtractSame(target_set,ignore);
609
610 same->SubtractAny(to_transfer);
611 }
612
613 if(newer_than!=NO_DATE)
614 to_transfer->SubtractNotNewerThan(newer_than);
615 if(older_than!=NO_DATE)
616 to_transfer->SubtractNotOlderThan(older_than);
617 if(size_range)
618 to_transfer->SubtractSizeOutside(size_range);
619
620 if(FlagSet(SCAN_ALL_FIRST)) {
621 to_mkdir=new FileSet(to_transfer);
622 to_mkdir->SubtractNotDirs();
623 to_mkdir->SubtractAny(target_set);
624 }
625
626 switch(recursion_mode) {
627 case RECURSION_NEVER:
628 to_transfer->SubtractDirs();
629 break;
630 case RECURSION_MISSING:
631 to_transfer->SubtractDirs(target_set);
632 break;
633 case RECURSION_NEWER:
634 to_transfer->SubtractNotOlderDirs(target_set);
635 break;
636 case RECURSION_ALWAYS:
637 break;
638 }
639
640 if(skip_noaccess)
641 to_transfer->ExcludeUnaccessible(source_session->GetUser());
642
643 new_files_set=new FileSet(to_transfer);
644 new_files_set->SubtractAny(target_set);
645 old_files_set=new FileSet(target_set);
646 old_files_set->SubtractNotIn(to_transfer);
647
648 to_rm_mismatched=new FileSet(old_files_set);
649 to_rm_mismatched->SubtractSameType(to_transfer);
650 to_rm_mismatched->SubtractNotDirs();
651
652 if(!FlagSet(DELETE))
653 to_transfer->SubtractAny(to_rm_mismatched);
654
655 if(FlagSet(TARGET_FLAT) && !parent_mirror && target_set) {
656 source_set->Unsort();
657 to_transfer->UnsortFlat();
658 to_transfer->SubtractDirs();
659 same->UnsortFlat();
660 to_mkdir->Empty();
661 new_files_set->UnsortFlat();
662 }
663
664 const char *sort_by=ResMgr::Query("mirror:sort-by",0);
665 bool desc=strstr(sort_by,"-desc");
666 if(!strncmp(sort_by,"name",4))
667 to_transfer->SortByPatternList(ResMgr::Query("mirror:order",0));
668 else if(!strncmp(sort_by,"date",4))
669 to_transfer->Sort(FileSet::BYDATE);
670 else if(!strncmp(sort_by,"size",4))
671 to_transfer->Sort(FileSet::BYSIZE,false,true);
672 if(desc)
673 to_transfer->ReverseSort();
674
675 int dir_count=0;
676 if(to_mkdir) {
677 to_mkdir->Count(&dir_count,NULL,NULL,NULL);
678 only_dirs = (dir_count==to_mkdir->count());
679 } else {
680 to_transfer->Count(&dir_count,NULL,NULL,NULL);
681 only_dirs = (dir_count==to_transfer->count());
682 }
683 }
684
ExcludeEmptyDir(const char * target_rel_dir)685 void MirrorJob::ExcludeEmptyDir(const char *target_rel_dir)
686 {
687 source_set->SubtractByName(basename_ptr(target_rel_dir));
688 }
689
690 /* root_transfer_count of child mirrors contains the value to add or
691 subtract from transfer_count when doing "cd", "mkdir", "ls" on the source
692 or target directories. This would prevent other mirrors (siblings, etc)
693 from starting more jobs.
694
695 root_transfer_count is initialized once in ctor, so that change of
696 mirror:parallel-directories setting won't disbalance the count.
697 */
MirrorStarted()698 void MirrorJob::MirrorStarted()
699 {
700 if(!parent_mirror)
701 return;
702 transfer_count+=root_transfer_count;
703 }
MirrorFinished()704 void MirrorJob::MirrorFinished()
705 {
706 if(!parent_mirror)
707 return;
708 assert(transfer_count>=root_transfer_count);
709 transfer_count-=root_transfer_count;
710 }
711
HandleChdir(FileAccessRef & session,int & redirections)712 void MirrorJob::HandleChdir(FileAccessRef& session, int &redirections)
713 {
714 if(!session->IsOpen())
715 return;
716 int res=session->Done();
717 if(res<0)
718 {
719 if(res==FA::NO_HOST)
720 {
721 eprintf("mirror: %s\n",session->StrError(res));
722 stats.error_count++;
723 MirrorFinished();
724 set_state(FINISHING);
725 return;
726 }
727 if(res==FA::FILE_MOVED)
728 {
729 // cd to another url.
730 const char *loc_c=session->GetNewLocation();
731 int max_redirections=ResMgr::Query("xfer:max-redirections",0);
732 if(loc_c && max_redirections>0)
733 {
734 if(++redirections>max_redirections)
735 goto cd_err_normal;
736 eprintf(_("%s: received redirection to `%s'\n"),"mirror",loc_c);
737
738 char *loc=alloca_strdup(loc_c);
739 ParsedURL u(loc,true);
740
741 bool is_file=(last_char(loc)!='/');
742 if(!u.proto)
743 {
744 FileAccess::Path new_cwd(session->GetNewCwd());
745 new_cwd.Change(0,is_file,loc);
746 session->PathVerify(new_cwd);
747 session->Roll();
748 return;
749 }
750 session->Close(); // loc_c is no longer valid.
751 session=FA::New(&u);
752 FileAccess::Path new_cwd(u.path,is_file,url::path_ptr(loc));
753 session->PathVerify(new_cwd);
754 return;
755 }
756 }
757 cd_err_normal:
758 if(session==target_session && (script_only || FlagSet(SCAN_ALL_FIRST)))
759 {
760 char *dir=alloca_strdup(session->GetFile());
761 session->Close();
762 session->Chdir(dir,false);
763 no_target_dir=true;
764 return;
765 }
766 if(session==source_session && create_target_dir
767 && !FlagSet(NO_EMPTY_DIRS) && !skip_noaccess && parent_mirror)
768 {
769 // create target dir even if failed to cd to source dir.
770 if(script)
771 fprintf(script,"mkdir %s\n",target_session->GetFileURL(target_dir).get());
772 if(!script_only)
773 {
774 ArgV *a=new ArgV("mkdir");
775 a->Append(target_dir);
776 mkdirJob *mkj=new mkdirJob(target_session->Clone(),a);
777 a->CombineTo(mkj->cmdline);
778 JobStarted(mkj);
779 }
780 }
781 remove_this_source_dir=false;
782 eprintf("mirror: %s\n",session->StrError(res));
783 stats.error_count++;
784 MirrorFinished();
785 set_state(FINISHING);
786 source_session->Close();
787 target_session->Close();
788 return;
789 }
790 if(res==FA::OK)
791 session->Close();
792 }
HandleListInfoCreation(const FileAccessRef & session,SMTaskRef<ListInfo> & list_info,const char * relative_dir)793 void MirrorJob::HandleListInfoCreation(const FileAccessRef& session,SMTaskRef<ListInfo>& list_info,const char *relative_dir)
794 {
795 if(state!=GETTING_LIST_INFO)
796 return;
797
798 if(session==target_session && no_target_dir)
799 {
800 target_set=new FileSet();
801 return;
802 }
803
804 list_info=session->MakeListInfo();
805 if(list_info==0)
806 {
807 eprintf(_("mirror: protocol `%s' is not suitable for mirror\n"),
808 session->GetProto());
809 MirrorFinished();
810 set_state(FINISHING);
811 return;
812 }
813 list_info->UseCache(use_cache);
814 int need=FileInfo::ALL_INFO;
815 if(FlagSet(IGNORE_TIME))
816 need&=~FileInfo::DATE;
817 if(FlagSet(IGNORE_SIZE))
818 need&=~FileInfo::SIZE;
819 list_info->Need(need);
820 if(FlagSet(RETR_SYMLINKS))
821 list_info->FollowSymlinks();
822
823 list_info->SetExclude(relative_dir,top_exclude?top_exclude:exclude);
824 list_info->Roll();
825 }
826
HandleListInfo(SMTaskRef<ListInfo> & list_info,Ref<FileSet> & set,Ref<FileSet> * fsx)827 void MirrorJob::HandleListInfo(SMTaskRef<ListInfo>& list_info, Ref<FileSet>& set, Ref<FileSet> *fsx)
828 {
829 if(!list_info)
830 return;
831 if(!list_info->Done())
832 return;
833 if(list_info->Error())
834 {
835 eprintf("mirror: %s\n",list_info->ErrorText());
836 stats.error_count++;
837 MirrorFinished();
838 set_state(FINISHING);
839 source_list_info=0;
840 target_list_info=0;
841 return;
842 }
843 set=list_info->GetResult();
844 if(fsx)
845 {
846 *fsx=list_info->GetExcluded();
847 (*fsx)->ExcludeDots();
848 }
849 list_info=0;
850 set->ExcludeDots(); // don't need .. and .
851 }
852
Do()853 int MirrorJob::Do()
854 {
855 int res;
856 int m=STALL;
857 FileInfo *file;
858 Job *j;
859
860 switch(state)
861 {
862 case(INITIAL_STATE):
863 remove_this_source_dir=(remove_source_dirs && source_dir.last_char()!='/');
864 if(!strcmp(target_dir,".") || !strcmp(target_dir,"..") || (FlagSet(SCAN_ALL_FIRST) && parent_mirror))
865 create_target_dir=false;
866
867 source_session->Chdir(source_dir);
868 source_redirections=0;
869 source_session->Roll();
870 set_state(CHANGING_DIR_SOURCE);
871 m=MOVED;
872 /*fallthrough*/
873 case(CHANGING_DIR_SOURCE):
874 HandleChdir(source_session,source_redirections);
875 if(state!=CHANGING_DIR_SOURCE)
876 return MOVED;
877 if(source_session->IsOpen())
878 return m;
879
880 source_dir.set(source_session->GetCwd().GetDirectory());
881
882 pre_MAKE_TARGET_DIR:
883 {
884 if(!create_target_dir)
885 goto pre_CHANGING_DIR_TARGET;
886 if(target_is_local)
887 {
888 struct stat st;
889 if((FlagSet(RETR_SYMLINKS)?stat:lstat)(target_dir,&st)!=-1)
890 {
891 if(S_ISDIR(st.st_mode))
892 {
893 // try to enable write access
894 // only if not enabled as chmod can clear sgid flags on directories
895 if(!script_only && (st.st_mode!=(st.st_mode|0700)))
896 chmod(target_dir,st.st_mode|0700);
897 create_target_dir=false;
898 goto pre_CHANGING_DIR_TARGET;
899 }
900 else
901 {
902 Report(_("Removing old local file `%s'"),target_dir.get());
903 if(script)
904 {
905 ArgV args("rm");
906 args.Append(target_session->GetFileURL(target_dir));
907 xstring_ca cmd(args.CombineQuoted());
908 fprintf(script,"%s\n",cmd.get());
909 }
910 if(!script_only)
911 {
912 if(remove(target_dir)==-1)
913 eprintf("mirror: remove(%s): %s\n",target_dir.get(),strerror(errno));
914 }
915 }
916 }
917 }
918
919 if(FlagSet(DEPTH_FIRST))
920 goto pre_GETTING_LIST_INFO;
921
922 if(target_relative_dir)
923 Report(_("Making directory `%s'"),target_relative_dir.get());
924 bool mkdir_p=(parent_mirror==0 || parent_mirror->create_target_dir);
925 if(script)
926 {
927 ArgV args("mkdir");
928 if(mkdir_p)
929 args.Append("-p");
930 args.Append(target_session->GetFileURL(target_dir));
931 xstring_ca cmd(args.CombineQuoted());
932 fprintf(script,"%s\n",cmd.get());
933 if(script_only)
934 goto pre_CHANGING_DIR_TARGET;
935 }
936 target_session->Mkdir(target_dir,mkdir_p);
937 set_state(MAKE_TARGET_DIR);
938 m=MOVED;
939 }
940 /*fallthrough*/
941 case(MAKE_TARGET_DIR):
942 res=target_session->Done();
943 if(res==FA::IN_PROGRESS)
944 return m;
945 target_session->Close();
946 create_target_dir=false;
947
948 pre_CHANGING_DIR_TARGET:
949 target_session->Chdir(target_dir);
950 target_redirections=0;
951 target_session->Roll();
952 set_state(CHANGING_DIR_TARGET);
953 m=MOVED;
954 /*fallthrough*/
955 case(CHANGING_DIR_TARGET):
956 HandleChdir(target_session,target_redirections);
957 if(state!=CHANGING_DIR_TARGET)
958 return MOVED;
959 if(target_session->IsOpen())
960 return m;
961 create_target_dir=false;
962
963 target_dir.set(target_session->GetCwd().GetDirectory());
964
965 pre_GETTING_LIST_INFO:
966 set_state(GETTING_LIST_INFO);
967 m=MOVED;
968 if(!source_set)
969 HandleListInfoCreation(source_session,source_list_info,source_relative_dir);
970 if(!target_set && !create_target_dir
971 && (!FlagSet(DEPTH_FIRST) || FlagSet(ONLY_EXISTING))
972 && !(FlagSet(TARGET_FLAT) && parent_mirror))
973 HandleListInfoCreation(target_session,target_list_info,target_relative_dir);
974 if(state!=GETTING_LIST_INFO)
975 {
976 source_list_info=0;
977 target_list_info=0;
978 }
979 return m; // give time to other tasks
980 case(GETTING_LIST_INFO):
981 HandleListInfo(source_list_info,source_set);
982 HandleListInfo(target_list_info,target_set,&target_set_excluded);
983 if(state!=GETTING_LIST_INFO)
984 return MOVED;
985 if(source_list_info || target_list_info)
986 return m;
987
988 MirrorFinished(); // leave room for transfers.
989
990 if(FlagSet(DEPTH_FIRST) && source_set && !target_set)
991 {
992 // transfer directories first
993 InitSets();
994 to_transfer->Unsort();
995 to_transfer->SubtractNotDirs();
996 goto pre_WAITING_FOR_TRANSFER;
997 }
998
999 // now we have both target and source file sets.
1000 if(parent_mirror)
1001 stats.dirs++;
1002
1003 if(FlagSet(SCAN_ALL_FIRST) && parent_mirror)
1004 {
1005 source_set->PrependPath(source_relative_dir);
1006 if(root_mirror->source_set_recursive)
1007 root_mirror->source_set_recursive->Merge(source_set);
1008 else
1009 root_mirror->source_set_recursive=source_set.borrow();
1010 if(target_set) {
1011 target_set->PrependPath(target_relative_dir);
1012 if(root_mirror->target_set_recursive)
1013 root_mirror->target_set_recursive->Merge(target_set);
1014 else
1015 root_mirror->target_set_recursive=target_set.borrow();
1016 }
1017 if(target_set_excluded) {
1018 target_set_excluded->PrependPath(target_relative_dir);
1019 if(root_mirror->target_set_excluded)
1020 root_mirror->target_set_excluded->Merge(target_set_excluded);
1021 else
1022 root_mirror->target_set_excluded=target_set_excluded.borrow();
1023 }
1024 root_mirror->stats.dirs++;
1025 transfer_count++; // parent mirror will decrement it.
1026 goto pre_DONE;
1027 }
1028
1029 if(source_set_recursive) {
1030 source_set->Merge(source_set_recursive);
1031 source_set_recursive=0;
1032 }
1033 if(target_set_recursive) {
1034 target_set->Merge(target_set_recursive);
1035 target_set_recursive=0;
1036 }
1037 InitSets();
1038
1039 to_transfer->CountBytes(&bytes_to_transfer);
1040 if(parent_mirror)
1041 parent_mirror->AddBytesToTransfer(bytes_to_transfer);
1042
1043 to_rm->Count(&stats.del_dirs,&stats.del_files,&stats.del_symlinks,&stats.del_files);
1044 to_rm->rewind();
1045 to_rm_mismatched->Count(&stats.del_dirs,&stats.del_files,&stats.del_symlinks,&stats.del_files);
1046 to_rm_mismatched->rewind();
1047
1048 target_set->Merge(target_set_excluded);
1049 target_set_excluded=0;
1050
1051 set_state(TARGET_REMOVE_OLD_FIRST);
1052 goto TARGET_REMOVE_OLD_FIRST_label;
1053
1054 pre_TARGET_MKDIR:
1055 if(!to_mkdir)
1056 goto pre_WAITING_FOR_TRANSFER;
1057 to_mkdir->rewind();
1058 set_state(TARGET_MKDIR);
1059 m=MOVED;
1060 /*fallthrough*/
1061 case(TARGET_MKDIR):
1062 while((j=FindDoneAwaitedJob())!=0)
1063 {
1064 JobFinished(j);
1065 m=MOVED;
1066 }
1067 if(max_error_count>0 && stats.error_count>=max_error_count)
1068 goto pre_FINISHING;
1069 while(transfer_count<parallel && state==TARGET_MKDIR)
1070 {
1071 file=to_mkdir->curr();
1072 if(!file)
1073 goto pre_WAITING_FOR_TRANSFER;
1074 to_mkdir->next();
1075 if(!file->TypeIs(file->DIRECTORY))
1076 continue;
1077 if(script)
1078 fprintf(script,"mkdir %s\n",target_session->GetFileURL(file->name).get());
1079 if(!script_only)
1080 {
1081 ArgV *a=new ArgV("mkdir");
1082 a->Append(file->name);
1083 mkdirJob *mkj=new mkdirJob(target_session->Clone(),a);
1084 a->CombineTo(mkj->cmdline);
1085 JobStarted(mkj);
1086 m=MOVED;
1087 }
1088 }
1089 break;
1090
1091 pre_WAITING_FOR_TRANSFER:
1092 to_transfer->rewind();
1093 set_state(WAITING_FOR_TRANSFER);
1094 m=MOVED;
1095 /*fallthrough*/
1096 case(WAITING_FOR_TRANSFER):
1097 while((j=FindDoneAwaitedJob())!=0)
1098 {
1099 TransferFinished(j);
1100 m=MOVED;
1101 }
1102 if(max_error_count>0 && stats.error_count>=max_error_count)
1103 goto pre_FINISHING;
1104 while(transfer_count<parallel && state==WAITING_FOR_TRANSFER)
1105 {
1106 file=to_transfer->curr();
1107 if(!file)
1108 {
1109 // go to the next step only when all transfers have finished
1110 if(waiting_num>0)
1111 break;
1112 if(FlagSet(DEPTH_FIRST))
1113 {
1114 // we have been in the depth, don't go there again
1115 SetFlags(DEPTH_FIRST,false);
1116 SetFlags(NO_RECURSION,true);
1117
1118 // if we have not created any subdirs and there are only subdirs,
1119 // then the directory would be empty - skip it.
1120 if(FlagSet(NO_EMPTY_DIRS) && stats.dirs==0 && only_dirs)
1121 {
1122 if(parent_mirror)
1123 parent_mirror->ExcludeEmptyDir(target_relative_dir);
1124 goto pre_FINISHING_FIX_LOCAL;
1125 }
1126
1127 MirrorStarted();
1128 goto pre_MAKE_TARGET_DIR;
1129 }
1130 goto pre_TARGET_REMOVE_OLD;
1131 }
1132 HandleFile(file);
1133 to_transfer->next();
1134 m=MOVED;
1135 }
1136 break;
1137
1138 pre_TARGET_REMOVE_OLD:
1139 if(FlagSet(REMOVE_FIRST))
1140 goto pre_TARGET_CHMOD;
1141 set_state(TARGET_REMOVE_OLD);
1142 m=MOVED;
1143 /*fallthrough*/
1144 case(TARGET_REMOVE_OLD):
1145 case(TARGET_REMOVE_OLD_FIRST):
1146 TARGET_REMOVE_OLD_FIRST_label:
1147 while((j=FindDoneAwaitedJob())!=0)
1148 {
1149 JobFinished(j);
1150 m=MOVED;
1151 }
1152 if(max_error_count>0 && stats.error_count>=max_error_count)
1153 goto pre_FINISHING;
1154 while(transfer_count<parallel && (state==TARGET_REMOVE_OLD || state==TARGET_REMOVE_OLD_FIRST))
1155 {
1156 file=0;
1157 if(!file && state==TARGET_REMOVE_OLD_FIRST)
1158 {
1159 file=to_rm_mismatched->curr();
1160 to_rm_mismatched->next();
1161 }
1162 if(!file && (state==TARGET_REMOVE_OLD || FlagSet(REMOVE_FIRST)))
1163 {
1164 file=to_rm->curr();
1165 to_rm->next();
1166 }
1167 if(!file)
1168 {
1169 if(waiting_num>0)
1170 break;
1171 if(state==TARGET_REMOVE_OLD)
1172 goto pre_TARGET_CHMOD;
1173 goto pre_TARGET_MKDIR;
1174 }
1175 if(!FlagSet(DELETE))
1176 {
1177 if(FlagSet(REPORT_NOT_DELETED))
1178 {
1179 const char *target_name_rel=dir_file(target_relative_dir,file->name);
1180 if(file->TypeIs(file->DIRECTORY))
1181 Report(_("Old directory `%s' is not removed"),target_name_rel);
1182 else
1183 Report(_("Old file `%s' is not removed"),target_name_rel);
1184 }
1185 continue;
1186 }
1187 bool use_rmdir = (file->TypeIs(file->DIRECTORY)
1188 && recursion_mode==RECURSION_NEVER);
1189 if(script)
1190 {
1191 ArgV args(use_rmdir?"rmdir":"rm");
1192 if(file->TypeIs(file->DIRECTORY) && !use_rmdir)
1193 args.Append("-r");
1194 args.Append(target_session->GetFileURL(file->name));
1195 xstring_ca cmd(args.CombineQuoted());
1196 fprintf(script,"%s\n",cmd.get());
1197 }
1198 if(!script_only)
1199 {
1200 ArgV *args=new ArgV(use_rmdir?"rmdir":"rm");
1201 args->Append(dir_file(".",file->name));
1202 args->seek(1);
1203 rmJob *j=new rmJob(target_session->Clone(),args);
1204 args->CombineTo(j->cmdline);
1205 JobStarted(j);
1206 if(file->TypeIs(file->DIRECTORY))
1207 {
1208 if(recursion_mode==RECURSION_NEVER)
1209 j->Rmdir();
1210 else
1211 j->Recurse();
1212 }
1213 }
1214 const char *target_name_rel=dir_file(target_relative_dir,file->name);
1215 if(file->TypeIs(file->DIRECTORY))
1216 Report(_("Removing old directory `%s'"),target_name_rel);
1217 else
1218 Report(_("Removing old file `%s'"),target_name_rel);
1219 }
1220 break;
1221
1222 pre_TARGET_CHMOD:
1223 if(FlagSet(NO_PERMS))
1224 goto pre_FINISHING_FIX_LOCAL;
1225
1226 to_transfer->rewind();
1227 if(FlagSet(TARGET_FLAT))
1228 to_transfer->Sort(FileSet::BYNAME_FLAT);
1229 set_state(TARGET_CHMOD);
1230 m=MOVED;
1231 /*fallthrough*/
1232 case(TARGET_CHMOD):
1233 while((j=FindDoneAwaitedJob())!=0)
1234 {
1235 JobFinished(j);
1236 m=MOVED;
1237 }
1238 if(max_error_count>0 && stats.error_count>=max_error_count)
1239 goto pre_FINISHING;
1240 while(transfer_count<parallel && state==TARGET_CHMOD)
1241 {
1242 file=to_transfer->curr();
1243 if(!file)
1244 goto pre_FINISHING_FIX_LOCAL;
1245 to_transfer->next();
1246 if(file->TypeIs(file->SYMLINK))
1247 continue;
1248 if(!file->Has(file->MODE))
1249 continue;
1250 mode_t mode_mask=get_mode_mask();
1251 mode_t def_mode=(file->TypeIs(file->DIRECTORY)?0775:0664)&~mode_mask;
1252 if(target_is_local && file->mode==def_mode)
1253 {
1254 struct stat st;
1255 if(!target_is_local || lstat(dir_file(target_dir,file->name),&st)==-1)
1256 continue;
1257 if((st.st_mode&07777)==(file->mode&~mode_mask))
1258 continue;
1259 }
1260 FileInfo *target=target_set->FindByName(file->name);
1261 if(target && target->filetype==file->DIRECTORY && file->filetype==file->DIRECTORY
1262 && target->mode==(file->mode&~mode_mask) && (target->mode&0200))
1263 continue;
1264 if(script)
1265 {
1266 ArgV args("chmod");
1267 args.Append("-f");
1268 args.Append(xstring::format("%03lo",(unsigned long)(file->mode&~mode_mask)));
1269 args.Append(target_session->GetFileURL(file->name));
1270 xstring_ca cmd(args.CombineQuoted());
1271 fprintf(script,"%s\n",cmd.get());
1272 }
1273 if(!script_only)
1274 {
1275 ArgV *a=new ArgV("chmod");
1276 a->Append(dir_file(".",file->name));
1277 a->seek(1);
1278 ChmodJob *cj=new ChmodJob(target_session->Clone(),
1279 file->mode&~mode_mask,a);
1280 a->CombineTo(cj->cmdline);
1281 if(!verbose_report)
1282 cj->BeQuiet(); // chmod is not supported on all servers; be quiet.
1283 JobStarted(cj);
1284 m=MOVED;
1285 }
1286 }
1287 break;
1288
1289 pre_FINISHING_FIX_LOCAL:
1290 if(target_is_local && !script_only) // FIXME
1291 {
1292 const bool flat=FlagSet(TARGET_FLAT);
1293 to_transfer->Sort(FileSet::BYNAME_FLAT);
1294 to_transfer->LocalUtime(target_dir,/*only_dirs=*/true,flat);
1295 if(FlagSet(ALLOW_CHOWN))
1296 to_transfer->LocalChown(target_dir,flat);
1297 if(!FlagSet(NO_PERMS) && same)
1298 same->LocalChmod(target_dir,get_mode_mask(),flat);
1299 if(FlagSet(ALLOW_CHOWN) && same)
1300 same->LocalChown(target_dir,flat);
1301 }
1302 if(remove_source_files && (same || to_rm_src))
1303 goto pre_SOURCE_REMOVING_SAME;
1304 pre_FINISHING:
1305 set_state(FINISHING);
1306 m=MOVED;
1307 /*fallthrough*/
1308 case(FINISHING):
1309 while((j=FindDoneAwaitedJob())!=0)
1310 {
1311 JobFinished(j);
1312 m=MOVED;
1313 }
1314 if(waiting_num>0)
1315 break;
1316
1317 // all jobs finished.
1318 if(remove_this_source_dir) {
1319 // remove source directory once.
1320 remove_this_source_dir=false;
1321 if(script)
1322 {
1323 ArgV args("rmdir");
1324 args.Append(source_session->GetFileURL(source_dir));
1325 xstring_ca cmd(args.CombineQuoted());
1326 fprintf(script,"%s\n",cmd.get());
1327 }
1328 if(!script_only)
1329 {
1330 ArgV *args=new ArgV("rmdir");
1331 args->Append(source_dir);
1332 args->seek(1);
1333 rmJob *j=new rmJob(source_session->Clone(),args);
1334 args->CombineTo(j->cmdline);
1335 j->Rmdir();
1336 JobStarted(j);
1337 }
1338 if(source_relative_dir)
1339 Report(_("Removing source directory `%s'"),source_relative_dir.get());
1340 m=MOVED;
1341 break;
1342 }
1343
1344 // all jobs finished and src dir removed, if needed.
1345
1346 transfer_count++; // parent mirror will decrement it.
1347 if(parent_mirror)
1348 parent_mirror->stats.Add(stats);
1349 else
1350 {
1351 if(stats.HaveSomethingDone(flags) && on_change)
1352 {
1353 CmdExec *exec=new CmdExec(GetExecSession().Clone(),0);
1354 AddWaiting(exec);
1355 exec->FeedCmd(on_change);
1356 exec->FeedCmd("\n");
1357 set_state(LAST_EXEC);
1358 break;
1359 }
1360 }
1361 goto pre_DONE;
1362
1363 pre_SOURCE_REMOVING_SAME:
1364 if(!same)
1365 same=to_rm_src.borrow();
1366 else if(to_rm_src)
1367 same->Merge(to_rm_src);
1368 same->rewind();
1369 set_state(SOURCE_REMOVING_SAME);
1370 m=MOVED;
1371 /*fallthrough*/
1372 case(SOURCE_REMOVING_SAME):
1373 while((j=FindDoneAwaitedJob())!=0)
1374 {
1375 JobFinished(j);
1376 m=MOVED;
1377 }
1378 if(max_error_count>0 && stats.error_count>=max_error_count)
1379 goto pre_FINISHING;
1380 while(transfer_count<parallel && state==SOURCE_REMOVING_SAME)
1381 {
1382 file=same->curr();
1383 same->next();
1384 if(!file)
1385 goto pre_FINISHING;
1386 if(file->TypeIs(file->DIRECTORY))
1387 continue;
1388 if(script)
1389 {
1390 ArgV args("rm");
1391 args.Append(source_session->GetFileURL(file->name));
1392 xstring_ca cmd(args.CombineQuoted());
1393 fprintf(script,"%s\n",cmd.get());
1394 }
1395 if(!script_only)
1396 {
1397 ArgV *args=new ArgV("rm");
1398 args->Append(dir_file(".",file->name));
1399 args->seek(1);
1400 rmJob *j=new rmJob(source_session->Clone(),args);
1401 args->CombineTo(j->cmdline);
1402 JobStarted(j);
1403 }
1404 const char *source_name_rel=dir_file(source_relative_dir,file->name);
1405 Report(_("Removing source file `%s'"),source_name_rel);
1406 }
1407 break;
1408
1409 case(LAST_EXEC):
1410 while((j=FindDoneAwaitedJob())!=0)
1411 {
1412 RemoveWaiting(j);
1413 Delete(j);
1414 m=MOVED;
1415 }
1416 if(waiting_num>0)
1417 break;
1418 pre_DONE:
1419 set_state(DONE);
1420 m=MOVED;
1421 bytes_transferred=0;
1422 if(!parent_mirror && FlagSet(LOOP) && stats.HaveSomethingDone(flags) && !stats.error_count)
1423 {
1424 PrintStatus(0,"");
1425 printf(_("Retrying mirror...\n"));
1426 stats.Reset();
1427 source_set=0;
1428 target_set=0;
1429 goto pre_GETTING_LIST_INFO;
1430 }
1431 /*fallthrough*/
1432 case(DONE):
1433 break;
1434 }
1435 // give direct parent priority over grand-parents.
1436 if(transfer_count<parallel && parent_mirror)
1437 m|=parent_mirror->Roll();
1438 return m;
1439 }
1440
MirrorJob(MirrorJob * parent,FileAccess * source,FileAccess * target,const char * new_source_dir,const char * new_target_dir)1441 MirrorJob::MirrorJob(MirrorJob *parent,
1442 FileAccess *source,FileAccess *target,
1443 const char *new_source_dir,const char *new_target_dir)
1444 :
1445 bytes_transferred(0), bytes_to_transfer(0),
1446 source_dir(new_source_dir), target_dir(new_target_dir),
1447 transfer_time_elapsed(0), root_transfer_count(0),
1448 verbose_report(0),
1449 parent_mirror(parent), root_mirror(parent?parent->root_mirror:this)
1450 {
1451
1452 source_session=source;
1453 target_session=target;
1454 // TODO: get rid of this.
1455 source_is_local=!strcmp(source_session->GetProto(),"file");
1456 target_is_local=!strcmp(target_session->GetProto(),"file");
1457
1458 create_target_dir=true;
1459 no_target_dir=false;
1460 remove_this_source_dir=false;
1461
1462 flags=0;
1463 recursion_mode=RECURSION_ALWAYS;
1464 max_error_count=0;
1465
1466 exclude=0;
1467
1468 set_state(INITIAL_STATE);
1469
1470 newer_than=NO_DATE;
1471 older_than=NO_DATE;
1472 size_range=0;
1473
1474 script=0;
1475 script_only=false;
1476 script_needs_closing=false;
1477
1478 use_cache=false;
1479 remove_source_files=false;
1480 remove_source_dirs=false;
1481 skip_noaccess=false;
1482
1483 parallel=1;
1484 pget_n=1;
1485 pget_minchunk=0x10000;
1486
1487 source_redirections=0;
1488 target_redirections=0;
1489
1490 if(parent_mirror)
1491 {
1492 bool parallel_dirs=ResMgr::QueryBool("mirror:parallel-directories",0);
1493 // If parallel_dirs is true, allow parent mirror to continue
1494 // processing other directories, otherwise block it until we
1495 // get file sets and start transfers.
1496 // See also comment at MirrorJob::MirrorStarted().
1497 root_transfer_count=parallel_dirs?1:1024;
1498
1499 // inherit flags and other things
1500 SetFlags(parent->flags,1);
1501 UseCache(parent->use_cache);
1502
1503 SetExclude(parent->exclude);
1504
1505 verbose_report=parent->verbose_report;
1506 newer_than=parent->newer_than;
1507 older_than=parent->older_than;
1508 size_range=parent->size_range;
1509 parallel=parent->parallel;
1510 pget_n=parent->pget_n;
1511 pget_minchunk=parent->pget_minchunk;
1512 remove_source_files=parent->remove_source_files;
1513 remove_source_dirs=parent->remove_source_dirs;
1514 skip_noaccess=parent->skip_noaccess;
1515 no_target_dir=parent->no_target_dir;
1516 recursion_mode=parent->recursion_mode;
1517
1518 script=parent->script;
1519 script_needs_closing=false;
1520 script_name.set(parent->script_name);
1521 script_only=parent->script_only;
1522
1523 max_error_count=parent->max_error_count;
1524 }
1525 MirrorStarted();
1526 }
1527
~MirrorJob()1528 MirrorJob::~MirrorJob()
1529 {
1530 if(script && script_needs_closing)
1531 fclose(script);
1532 }
1533
va_Report(const char * fmt,va_list v)1534 void MirrorJob::va_Report(const char *fmt,va_list v)
1535 {
1536 if(parent_mirror)
1537 {
1538 parent_mirror->va_Report(fmt,v);
1539 return;
1540 }
1541
1542 if(verbose_report)
1543 {
1544 pid_t p=tcgetpgrp(fileno(stdout));
1545 if(p>0 && p!=getpgrp())
1546 return;
1547
1548 vfprintf(stdout,fmt,v);
1549 printf("\n");
1550 fflush(stdout);
1551 }
1552 }
1553
Report(const char * fmt,...)1554 void MirrorJob::Report(const char *fmt,...)
1555 {
1556 va_list v;
1557 va_start(v,fmt);
1558
1559 va_Report(fmt,v);
1560
1561 va_end(v);
1562 }
1563
1564 extern "C" {
1565 #include "parse-datetime.h"
1566 }
SetNewerThan(const char * f)1567 void MirrorJob::SetNewerThan(const char *f)
1568 {
1569 struct timespec ts;
1570 if(parse_datetime(&ts,f,0))
1571 {
1572 newer_than=ts.tv_sec;
1573 return;
1574 }
1575 struct stat st;
1576 if(stat(f,&st)==-1)
1577 {
1578 perror(f);
1579 return;
1580 }
1581 newer_than=st.st_mtime;
1582 }
SetOlderThan(const char * f)1583 void MirrorJob::SetOlderThan(const char *f)
1584 {
1585 struct timespec ts;
1586 if(parse_datetime(&ts,f,0))
1587 {
1588 older_than=ts.tv_sec;
1589 return;
1590 }
1591 struct stat st;
1592 if(stat(f,&st)==-1)
1593 {
1594 perror(f);
1595 return;
1596 }
1597 older_than=st.st_mtime;
1598 }
1599
get_mode_mask()1600 mode_t MirrorJob::get_mode_mask()
1601 {
1602 mode_t mode_mask=0;
1603 if(!FlagSet(ALLOW_SUID))
1604 mode_mask|=S_ISUID|S_ISGID;
1605 if(!FlagSet(NO_UMASK))
1606 {
1607 if(target_is_local)
1608 {
1609 mode_t u=umask(022); // get+set
1610 umask(u); // retore
1611 mode_mask|=u;
1612 }
1613 else
1614 mode_mask|=022; // sane default.
1615 }
1616 return mode_mask;
1617 }
1618
Fg()1619 void MirrorJob::Fg()
1620 {
1621 Job::Fg();
1622 source_session->SetPriority(1);
1623 target_session->SetPriority(1);
1624 }
Bg()1625 void MirrorJob::Bg()
1626 {
1627 source_session->SetPriority(0);
1628 target_session->SetPriority(0);
1629 Job::Bg();
1630 }
1631
Statistics()1632 MirrorJob::Statistics::Statistics()
1633 {
1634 Reset();
1635 error_count=0;
1636 bytes=0;
1637 time=0;
1638 }
Reset()1639 void MirrorJob::Statistics::Reset()
1640 {
1641 tot_files=new_files=mod_files=del_files=
1642 tot_symlinks=new_symlinks=mod_symlinks=del_symlinks=
1643 dirs=del_dirs=0;
1644 }
Add(const Statistics & s)1645 void MirrorJob::Statistics::Add(const Statistics &s)
1646 {
1647 tot_files +=s.tot_files;
1648 new_files +=s.new_files;
1649 mod_files +=s.mod_files;
1650 del_files +=s.del_files;
1651 tot_symlinks+=s.tot_symlinks;
1652 new_symlinks+=s.new_symlinks;
1653 mod_symlinks+=s.mod_symlinks;
1654 del_symlinks+=s.del_symlinks;
1655 dirs +=s.dirs;
1656 del_dirs +=s.del_dirs;
1657 error_count +=s.error_count;
1658 bytes +=s.bytes;
1659 time +=s.time;
1660 }
HaveSomethingDone(unsigned flags)1661 bool MirrorJob::Statistics::HaveSomethingDone(unsigned flags)
1662 {
1663 bool del=(flags&MirrorJob::DELETE);
1664 return new_files|mod_files|(del_files*del)|new_symlinks|mod_symlinks|(del_symlinks*del)|(del_dirs*del);
1665 }
1666
SetScriptFile(const char * n)1667 const char *MirrorJob::SetScriptFile(const char *n)
1668 {
1669 script_name.set(n);
1670 if(strcmp(n,"-"))
1671 {
1672 script=fopen(n,"w");
1673 if(!script)
1674 return xstring::format("%s: %s",n,strerror(errno));
1675 setvbuf(script,NULL,_IOLBF,0);
1676 script_needs_closing=true;
1677 }
1678 else
1679 {
1680 script=stdout;
1681 script_needs_closing=false;
1682 }
1683 return 0;
1684 }
1685
SetOnChange(const char * oc)1686 void MirrorJob::SetOnChange(const char *oc)
1687 {
1688 on_change.set(oc);
1689 }
1690
AddPattern(Ref<PatternSet> & exclude,char opt,const char * optarg)1691 const char *MirrorJob::AddPattern(Ref<PatternSet>& exclude,char opt,const char *optarg)
1692 {
1693 if(!optarg || !*optarg)
1694 return _("pattern is empty");
1695
1696 PatternSet::Type type=
1697 (opt=='x'||opt=='X'||opt=='\0'?PatternSet::EXCLUDE:PatternSet::INCLUDE);
1698 PatternSet::Pattern *pattern=0;
1699 if(opt=='x' || opt=='i')
1700 {
1701 Ref<PatternSet::Regex> rx(new PatternSet::Regex(optarg));
1702 if(rx->Error())
1703 return xstring::get_tmp(rx->ErrorText());
1704 pattern=rx.borrow();
1705 }
1706 else if(opt=='X' || opt=='I')
1707 {
1708 pattern=new PatternSet::Glob(optarg);
1709 }
1710 if(!exclude)
1711 {
1712 const char *default_exclude=ResMgr::Query("mirror:exclude-regex",0);
1713 const char *default_include=ResMgr::Query("mirror:include-regex",0);
1714
1715 // don't create default pattern set if not needed
1716 if(!pattern && !(default_exclude && *default_exclude))
1717 return NULL;
1718
1719 exclude=new PatternSet;
1720 /* Make default_exclude the first pattern so that it can be
1721 * overridden by --include later, and do that only when first
1722 * explicit pattern is for exclusion - otherwise all files are
1723 * excluded by default and no default exclusion is needed. */
1724 if(type==PatternSet::EXCLUDE && default_exclude && *default_exclude)
1725 {
1726 exclude->Add(type,new PatternSet::Regex(default_exclude));
1727 if(default_include && *default_include)
1728 exclude->Add(PatternSet::INCLUDE,new PatternSet::Regex(default_include));
1729 }
1730 }
1731 if(pattern)
1732 exclude->Add(type,pattern);
1733
1734 return NULL; // no error
1735 }
1736
AddPatternsFrom(Ref<PatternSet> & exclude,char opt,const char * file)1737 const char *MirrorJob::AddPatternsFrom(Ref<PatternSet>& exclude,char opt,const char *file)
1738 {
1739 FILE *f=fopen(file,"r");
1740 if(!f)
1741 return xstring::format("%s: %s",file,strerror(errno));
1742
1743 xstring line;
1744 const char *err=0;
1745 int c;
1746 while(!feof(f)) {
1747 line.truncate();
1748 while((c=getc(f))!=EOF && c!='\n')
1749 line.append(c);
1750 if(line.length()>0) {
1751 err=AddPattern(exclude,opt,line);
1752 if(err)
1753 break;
1754 }
1755 }
1756
1757 fclose(f);
1758 return err;
1759 }
1760
SetRecursionMode(const char * m)1761 const char *MirrorJob::SetRecursionMode(const char *m)
1762 {
1763 struct { const char name[8]; recursion_mode_t mode; } map[]={
1764 {"always", RECURSION_ALWAYS},
1765 {"never", RECURSION_NEVER},
1766 {"missing",RECURSION_MISSING},
1767 {"newer", RECURSION_NEWER},
1768 };
1769 unsigned i;
1770 for(i=0; i<sizeof(map)/sizeof(map[0]); i++) {
1771 if(!strcasecmp(m,map[i].name)) {
1772 recursion_mode=map[i].mode;
1773 return 0;
1774 }
1775 }
1776 xstring list(map[0].name);
1777 for(i=1; i<sizeof(map)/sizeof(map[0]); i++)
1778 list.append(", ").append(map[i].name);
1779 return xstring::format(_("%s must be one of: %s"),"--recursion",list.get());
1780 }
1781
CMD(mirror)1782 CMD(mirror)
1783 {
1784 #define args (parent->args)
1785 #define eprintf parent->eprintf
1786 enum {
1787 OPT_ALLOW_CHOWN,
1788 OPT_DELETE_FIRST,
1789 OPT_IGNORE_SIZE,
1790 OPT_IGNORE_TIME,
1791 OPT_LOOP,
1792 OPT_MAX_ERRORS,
1793 OPT_NO_DEREFERENCE,
1794 OPT_NO_SYMLINKS,
1795 OPT_NO_UMASK,
1796 OPT_OLDER_THAN,
1797 OPT_ONLY_MISSING,
1798 OPT_ONLY_EXISTING,
1799 OPT_PERMS,
1800 OPT_REMOVE_SOURCE_FILES,
1801 OPT_REMOVE_SOURCE_DIRS,
1802 OPT_SCRIPT,
1803 OPT_SCRIPT_ONLY,
1804 OPT_SIZE_RANGE,
1805 OPT_USE_CACHE,
1806 OPT_USE_PGET_N,
1807 OPT_SKIP_NOACCESS,
1808 OPT_ON_CHANGE,
1809 OPT_NO_EMPTY_DIRS,
1810 OPT_DEPTH_FIRST,
1811 OPT_ASCII,
1812 OPT_SCAN_ALL_FIRST,
1813 OPT_OVERWRITE,
1814 OPT_NO_OVERWRITE,
1815 OPT_RECURSION,
1816 OPT_UPLOAD_OLDER,
1817 OPT_TRANSFER_ALL,
1818 OPT_TARGET_FLAT,
1819 OPT_DELETE_EXCLUDED,
1820 };
1821 static const struct option mirror_opts[]=
1822 {
1823 {"delete",no_argument,0,'e'},
1824 {"allow-suid",no_argument,0,'s'},
1825 {"allow-chown",no_argument,0,OPT_ALLOW_CHOWN},
1826 {"include",required_argument,0,'i'},
1827 {"exclude",required_argument,0,'x'},
1828 {"include-glob",required_argument,0,'I'},
1829 {"exclude-glob",required_argument,0,'X'},
1830 {"include-rx-from",required_argument,0,'i'+'f'},
1831 {"exclude-rx-from",required_argument,0,'x'+'f'},
1832 {"include-glob-from",required_argument,0,'I'+'f'},
1833 {"exclude-glob-from",required_argument,0,'X'+'f'},
1834 {"only-newer",no_argument,0,'n'},
1835 {"no-recursion",no_argument,0,'r'},
1836 {"no-perms",no_argument,0,'p'},
1837 {"perms",no_argument,0,OPT_PERMS},
1838 {"no-umask",no_argument,0,OPT_NO_UMASK},
1839 {"continue",no_argument,0,'c'},
1840 {"reverse",no_argument,0,'R'},
1841 {"verbose",optional_argument,0,'v'},
1842 {"newer-than",required_argument,0,'N'},
1843 {"file",required_argument,0,'f'},
1844 {"directory",required_argument,0,'F'},
1845 {"older-than",required_argument,0,OPT_OLDER_THAN},
1846 {"size-range",required_argument,0,OPT_SIZE_RANGE},
1847 {"dereference",no_argument,0,'L'},
1848 {"no-dereference",no_argument,0,OPT_NO_DEREFERENCE},
1849 {"use-cache",no_argument,0,OPT_USE_CACHE},
1850 {"Remove-source-files",no_argument,0,OPT_REMOVE_SOURCE_FILES},
1851 {"Remove-source-dirs",no_argument,0,OPT_REMOVE_SOURCE_DIRS},
1852 {"Move",no_argument,0,OPT_REMOVE_SOURCE_DIRS},
1853 {"parallel",optional_argument,0,'P'},
1854 {"ignore-time",no_argument,0,OPT_IGNORE_TIME},
1855 {"ignore-size",no_argument,0,OPT_IGNORE_SIZE},
1856 {"only-missing",no_argument,0,OPT_ONLY_MISSING},
1857 {"only-existing",no_argument,0,OPT_ONLY_EXISTING},
1858 {"log",required_argument,0,OPT_SCRIPT},
1859 {"script", required_argument,0,OPT_SCRIPT_ONLY},
1860 {"just-print",optional_argument,0,OPT_SCRIPT_ONLY},
1861 {"dry-run", optional_argument,0,OPT_SCRIPT_ONLY},
1862 {"delete-first",no_argument,0,OPT_DELETE_FIRST},
1863 {"use-pget-n",optional_argument,0,OPT_USE_PGET_N},
1864 {"no-symlinks",no_argument,0,OPT_NO_SYMLINKS},
1865 {"loop",no_argument,0,OPT_LOOP},
1866 {"max-errors",required_argument,0,OPT_MAX_ERRORS},
1867 {"skip-noaccess",no_argument,0,OPT_SKIP_NOACCESS},
1868 {"on-change",required_argument,0,OPT_ON_CHANGE},
1869 {"no-empty-dirs",no_argument,0,OPT_NO_EMPTY_DIRS},
1870 {"depth-first",no_argument,0,OPT_DEPTH_FIRST},
1871 {"ascii",no_argument,0,OPT_ASCII},
1872 {"target-directory",required_argument,0,'O'},
1873 {"destination-directory",required_argument,0,'O'},
1874 {"scan-all-first",no_argument,0,OPT_SCAN_ALL_FIRST},
1875 {"overwrite",no_argument,0,OPT_OVERWRITE},
1876 {"no-overwrite",no_argument,0,OPT_NO_OVERWRITE},
1877 {"recursion",required_argument,0,OPT_RECURSION},
1878 {"upload-older",no_argument,0,OPT_UPLOAD_OLDER},
1879 {"transfer-all",no_argument,0,OPT_TRANSFER_ALL},
1880 {"flat",no_argument,0,OPT_TARGET_FLAT},
1881 {"delete-excluded",no_argument,0,OPT_DELETE_EXCLUDED},
1882 {0}
1883 };
1884
1885 int opt;
1886 unsigned flags=0;
1887 int max_error_count=0;
1888
1889 bool use_cache=false;
1890
1891 FileAccessRef source_session;
1892 FileAccessRef target_session;
1893
1894 int verbose=0;
1895 const char *newer_than=0;
1896 const char *older_than=0;
1897 Ref<Range> size_range;
1898 bool remove_source_files=false;
1899 bool remove_source_dirs=false;
1900 bool skip_noaccess=ResMgr::QueryBool("mirror:skip-noaccess",0);
1901 int parallel=-1;
1902 int use_pget=-1;
1903 bool reverse=false;
1904 bool script_only=false;
1905 bool no_empty_dirs=ResMgr::QueryBool("mirror:no-empty-dirs",0);
1906 const char *script_file=0;
1907 const char *on_change=0;
1908 const char *recursion_mode=0;
1909 bool single_file=false;
1910 bool single_dir=false;
1911
1912 Ref<PatternSet> exclude;
1913 Ref<PatternSet> top_exclude;
1914
1915 if(!ResMgr::QueryBool("mirror:set-permissions",0))
1916 flags|=MirrorJob::NO_PERMS;
1917 if(ResMgr::QueryBool("mirror:dereference",0))
1918 flags|=MirrorJob::RETR_SYMLINKS;
1919 if(ResMgr::QueryBool("mirror:overwrite",0))
1920 flags|=MirrorJob::OVERWRITE;
1921
1922 const char *source_dir=NULL;
1923 const char *target_dir=NULL;
1924
1925 args->rewind();
1926 while((opt=args->getopt_long("esi:x:I:X:nrpcRvN:LP:af:F:O:",mirror_opts,0))!=EOF)
1927 {
1928 switch(opt)
1929 {
1930 case('e'):
1931 flags|=MirrorJob::DELETE;
1932 break;
1933 case('s'):
1934 flags|=MirrorJob::ALLOW_SUID;
1935 break;
1936 case(OPT_ALLOW_CHOWN):
1937 flags|=MirrorJob::ALLOW_CHOWN;
1938 break;
1939 case('a'):
1940 flags|=MirrorJob::ALLOW_SUID|MirrorJob::ALLOW_CHOWN|MirrorJob::NO_UMASK;
1941 break;
1942 case('r'):
1943 recursion_mode="never";
1944 break;
1945 case('n'):
1946 flags|=MirrorJob::ONLY_NEWER;
1947 break;
1948 case('p'):
1949 flags|=MirrorJob::NO_PERMS;
1950 break;
1951 case(OPT_PERMS):
1952 flags&=~MirrorJob::NO_PERMS;
1953 break;
1954 case('c'):
1955 flags|=MirrorJob::CONTINUE;
1956 break;
1957 case('x'):
1958 case('i'):
1959 case('X'):
1960 case('I'):
1961 {
1962 const char *err=MirrorJob::AddPattern(exclude,opt,optarg);
1963 if(err)
1964 {
1965 eprintf("%s: %s\n",args->a0(),err);
1966 goto no_job;
1967 }
1968 break;
1969 }
1970 case('x'+'f'):
1971 case('i'+'f'):
1972 case('X'+'f'):
1973 case('I'+'f'):
1974 {
1975 const char *err=MirrorJob::AddPatternsFrom(exclude,opt-'f',optarg);
1976 if(err)
1977 {
1978 eprintf("%s: %s\n",args->a0(),err);
1979 goto no_job;
1980 }
1981 break;
1982 }
1983 case('R'):
1984 reverse=true;
1985 break;
1986 case('L'):
1987 flags|=MirrorJob::RETR_SYMLINKS;
1988 break;
1989 case(OPT_NO_DEREFERENCE):
1990 flags&=~MirrorJob::RETR_SYMLINKS;
1991 break;
1992 case('v'):
1993 if(optarg)
1994 verbose=atoi(optarg);
1995 else
1996 verbose++;
1997 if(verbose>1)
1998 flags|=MirrorJob::REPORT_NOT_DELETED;
1999 break;
2000 case('N'):
2001 newer_than=optarg;
2002 break;
2003 case('f'): // mirror for a single file (or glob pattern).
2004 single_file=true;
2005 /*fallthrough*/
2006 case('F'): // mirror for a single directory (or glob pattern).
2007 {
2008 xstring pattern(basename_ptr(optarg));
2009 if(opt=='F') {
2010 single_dir=true;
2011 if(pattern.last_char()!='/')
2012 pattern.append('/');
2013 }
2014 if(!top_exclude)
2015 top_exclude=new PatternSet();
2016 top_exclude->Add(PatternSet::INCLUDE,new PatternSet::Glob(pattern));
2017 const char *dir=dirname(optarg);
2018 if(source_dir && strcmp(source_dir,dir)) {
2019 eprintf(_("%s: multiple --file or --directory options must have the same base directory\n"),args->a0());
2020 goto no_job;
2021 }
2022 if(!source_dir)
2023 source_dir=alloca_strdup(dir); // save the temp string
2024 break;
2025 }
2026 case('O'):
2027 target_dir=optarg;
2028 break;
2029 case(OPT_OLDER_THAN):
2030 older_than=optarg;
2031 break;
2032 case(OPT_SIZE_RANGE):
2033 size_range=new Range(optarg);
2034 if(size_range->Error())
2035 {
2036 eprintf("%s: --size-range \"%s\": %s\n",
2037 args->a0(),optarg,size_range->ErrorText());
2038 goto no_job;
2039 }
2040 break;
2041 case(OPT_NO_UMASK):
2042 flags|=MirrorJob::NO_UMASK;
2043 break;
2044 case(OPT_USE_CACHE):
2045 use_cache=true;
2046 break;
2047 case(OPT_REMOVE_SOURCE_FILES):
2048 remove_source_files=true;
2049 break;
2050 case(OPT_REMOVE_SOURCE_DIRS):
2051 remove_source_dirs=true;
2052 break;
2053 case(OPT_IGNORE_TIME):
2054 flags|=MirrorJob::IGNORE_TIME;
2055 break;
2056 case(OPT_IGNORE_SIZE):
2057 flags|=MirrorJob::IGNORE_SIZE;
2058 break;
2059 case(OPT_ONLY_MISSING):
2060 flags|=MirrorJob::IGNORE_TIME|MirrorJob::IGNORE_SIZE;
2061 break;
2062 case('P'):
2063 if(optarg)
2064 parallel=atoi(optarg);
2065 else
2066 parallel=3;
2067 break;
2068 case(OPT_USE_PGET_N):
2069 if(optarg)
2070 use_pget=atoi(optarg);
2071 else
2072 use_pget=3;
2073 break;
2074 case(OPT_SCRIPT_ONLY):
2075 script_only=true;
2076 case(OPT_SCRIPT):
2077 script_file=optarg;
2078 if(script_file==0)
2079 script_file="-";
2080 break;
2081 case(OPT_DELETE_FIRST):
2082 flags|=MirrorJob::REMOVE_FIRST|MirrorJob::DELETE;
2083 break;
2084 case(OPT_NO_SYMLINKS):
2085 flags|=MirrorJob::NO_SYMLINKS;
2086 break;
2087 case(OPT_LOOP):
2088 flags|=MirrorJob::LOOP;
2089 break;
2090 case(OPT_MAX_ERRORS):
2091 max_error_count=atoi(optarg);
2092 break;
2093 case(OPT_SKIP_NOACCESS):
2094 skip_noaccess=true;
2095 break;
2096 case(OPT_ON_CHANGE):
2097 on_change=optarg;
2098 break;
2099 case(OPT_ONLY_EXISTING):
2100 flags|=MirrorJob::ONLY_EXISTING;
2101 break;
2102 case(OPT_NO_EMPTY_DIRS):
2103 no_empty_dirs=true;
2104 break;
2105 case(OPT_DEPTH_FIRST):
2106 flags|=MirrorJob::DEPTH_FIRST;
2107 break;
2108 case(OPT_SCAN_ALL_FIRST):
2109 flags|=MirrorJob::SCAN_ALL_FIRST|MirrorJob::DEPTH_FIRST;
2110 break;
2111 case(OPT_ASCII):
2112 flags|=MirrorJob::ASCII|MirrorJob::IGNORE_SIZE;
2113 break;
2114 case(OPT_OVERWRITE):
2115 flags|=MirrorJob::OVERWRITE;
2116 break;
2117 case(OPT_NO_OVERWRITE):
2118 flags&=~MirrorJob::OVERWRITE;
2119 break;
2120 case(OPT_RECURSION):
2121 recursion_mode=optarg;
2122 break;
2123 case(OPT_UPLOAD_OLDER):
2124 flags|=MirrorJob::UPLOAD_OLDER;
2125 break;
2126 case(OPT_TRANSFER_ALL):
2127 flags|=MirrorJob::TRANSFER_ALL;
2128 break;
2129 case(OPT_TARGET_FLAT):
2130 flags|=MirrorJob::TARGET_FLAT|MirrorJob::SCAN_ALL_FIRST|MirrorJob::DEPTH_FIRST;
2131 break;
2132 case(OPT_DELETE_EXCLUDED):
2133 flags|=MirrorJob::DELETE_EXCLUDED;
2134 break;
2135 case('?'):
2136 eprintf(_("Try `help %s' for more information.\n"),args->a0());
2137 no_job:
2138 return 0;
2139 }
2140 }
2141
2142 if(exclude && xstrcasecmp(recursion_mode,"never"))
2143 {
2144 /* Users usually don't want to exclude all directories when recursing */
2145 if(exclude->GetFirstType()==PatternSet::INCLUDE)
2146 exclude->AddFirst(PatternSet::INCLUDE,new PatternSet::Regex("/$"));
2147 }
2148
2149 /* add default exclusion if no explicit patterns were specified */
2150 if(!exclude)
2151 MirrorJob::AddPattern(exclude,'\0',0);
2152
2153 args->back();
2154
2155 const char *arg=args->getnext();
2156 if(arg)
2157 {
2158 if(source_dir)
2159 {
2160 eprintf(_("%s: ambiguous source directory (`%s' or `%s'?)\n"),args->a0(),
2161 source_dir,arg);
2162 goto no_job;
2163 }
2164 source_dir=arg;
2165 ParsedURL source_url(source_dir);
2166 if(source_url.proto && source_url.path)
2167 {
2168 source_session=FileAccess::New(&source_url);
2169 if(!source_session)
2170 {
2171 eprintf("%s: %s%s\n",args->a0(),source_url.proto.get(),
2172 _(" - not supported protocol"));
2173 goto no_job;
2174 }
2175 source_dir=alloca_strdup(source_url.path);
2176 }
2177 arg=args->getnext();
2178 if(arg)
2179 {
2180 if(target_dir)
2181 {
2182 eprintf(_("%s: ambiguous target directory (`%s' or `%s'?)\n"),args->a0(),
2183 target_dir,arg);
2184 goto no_job;
2185 }
2186 target_dir=arg;
2187 ParsedURL target_url(target_dir);
2188 if(target_url.proto && target_url.path)
2189 {
2190 target_session=FileAccess::New(&target_url);
2191 if(!target_session)
2192 {
2193 eprintf("%s: %s%s\n",args->a0(),target_url.proto.get(),
2194 _(" - not supported protocol"));
2195 goto no_job;
2196 }
2197 target_dir=alloca_strdup(target_url.path);
2198 }
2199 if(last_char(arg)=='/' && basename_ptr(arg)[0]!='/' && last_char(source_dir)!='/')
2200 {
2201 // user wants source dir name appended.
2202 const char *base=basename_ptr(source_dir);
2203 if(base[0]!='/' && strcmp(base,basename_ptr(arg))) {
2204 target_dir=xstring::cat(target_dir,base,NULL);
2205 target_dir=alloca_strdup(target_dir); // save the buffer
2206 }
2207 }
2208 }
2209 else
2210 {
2211 target_dir=basename_ptr(source_dir);
2212 if(target_dir[0]=='/')
2213 target_dir=".";
2214 else if(target_dir[0]=='~') {
2215 target_dir=dir_file(".",target_dir);
2216 target_dir=alloca_strdup(target_dir); // save the buffer
2217 }
2218 }
2219 }
2220
2221 if(!source_dir) {
2222 if(ResMgr::QueryBool("mirror:require-source",0)) {
2223 eprintf(_("%s: source directory is required (mirror:require-source is set)\n"),args->a0());
2224 return 0;
2225 }
2226 source_dir=".";
2227 }
2228 if(!target_dir)
2229 target_dir=".";
2230
2231 if(!reverse)
2232 {
2233 if(!source_session)
2234 source_session=parent->session->Clone();
2235 if(!target_session)
2236 target_session=FileAccess::New("file");
2237 }
2238 else //reverse
2239 {
2240 if(!source_session)
2241 source_session=FileAccess::New("file");
2242 if(!target_session)
2243 target_session=parent->session->Clone();
2244 flags|=MirrorJob::REVERSE;
2245 }
2246
2247 if(no_empty_dirs)
2248 flags|=MirrorJob::NO_EMPTY_DIRS|MirrorJob::DEPTH_FIRST;
2249
2250 if(recursion_mode && strcasecmp(recursion_mode,"always")
2251 && (flags&MirrorJob::DEPTH_FIRST)) {
2252 eprintf("%s: --recursion-mode=%s conflicts with other specified options\n",
2253 args->a0(),recursion_mode);
2254 return 0;
2255 }
2256
2257 if(parallel<0) {
2258 int parallel1=ResMgr::Query("mirror:parallel-transfer-count",source_session->GetHostName());
2259 int parallel2=ResMgr::Query("mirror:parallel-transfer-count",target_session->GetHostName());
2260 if(parallel1>0)
2261 parallel=parallel1;
2262 if(parallel2>0 && (parallel<0 || parallel>parallel2))
2263 parallel=parallel2;
2264 }
2265 if(use_pget<0) {
2266 int use_pget1=ResMgr::Query("mirror:use-pget-n",source_session->GetHostName());
2267 int use_pget2=ResMgr::Query("mirror:use-pget-n",target_session->GetHostName());
2268 if(use_pget1>0)
2269 use_pget=use_pget1;
2270 if(use_pget2>0 && (use_pget<0 || use_pget>use_pget2))
2271 use_pget=use_pget2;
2272 }
2273
2274 JobRef<MirrorJob> j(new MirrorJob(0,source_session.borrow(),target_session.borrow(),source_dir,target_dir));
2275 j->SetFlags(flags,1);
2276 j->SetVerbose(verbose);
2277 j->SetExclude(exclude.borrow());
2278 j->SetTopExclude(top_exclude.borrow());
2279
2280 if(newer_than)
2281 j->SetNewerThan(newer_than);
2282 if(older_than)
2283 j->SetOlderThan(older_than);
2284 if(size_range)
2285 j->SetSizeRange(size_range.borrow());
2286 j->UseCache(use_cache);
2287 if(remove_source_files)
2288 j->RemoveSourceFiles();
2289 if(remove_source_dirs)
2290 j->RemoveSourceDirs();
2291 if(skip_noaccess)
2292 j->SkipNoAccess();
2293 if(parallel<0)
2294 parallel=0;
2295 if(parallel>64)
2296 parallel=64; // a (in)sane limit.
2297 if(parallel)
2298 j->SetParallel(parallel);
2299 if(use_pget>1 && !(flags&MirrorJob::ASCII))
2300 j->SetPGet(use_pget);
2301
2302 if(!recursion_mode && single_file && !single_dir)
2303 recursion_mode="never";
2304
2305 if(recursion_mode) {
2306 const char *err=j->SetRecursionMode(recursion_mode);
2307 if(err) {
2308 eprintf("%s: %s\n",args->a0(),err);
2309 return 0;
2310 }
2311 }
2312 if(script_file)
2313 {
2314 const char *err=j->SetScriptFile(script_file);
2315 if(err)
2316 {
2317 eprintf("%s: %s\n",args->a0(),err);
2318 return 0;
2319 }
2320 }
2321 if(script_only)
2322 {
2323 j->ScriptOnly();
2324 if(!script_file)
2325 j->SetScriptFile("-");
2326 }
2327 j->SetMaxErrorCount(max_error_count);
2328 if(on_change)
2329 j->SetOnChange(on_change);
2330
2331 return j.borrow();
2332
2333 #undef args
2334 }
2335
2336 #include "modconfig.h"
2337 #ifndef MODULE_CMD_MIRROR
2338 # define module_init cmd_mirror_module_init
2339 #endif
module_init()2340 CDECL void module_init()
2341 {
2342 CmdExec::RegisterCommand("mirror",cmd_mirror,0,
2343 N_("\n"
2344 "Mirror specified remote directory to local directory\n"
2345 "\n"
2346 " -R, --reverse reverse mirror (put files)\n"
2347 "Lots of other options are documented in the man page lftp(1).\n"
2348 "\n"
2349 "When using -R, the first directory is local and the second is remote.\n"
2350 "If the second directory is omitted, basename of the first directory is used.\n"
2351 "If both directories are omitted, current local and remote directories are used.\n"
2352 "\n"
2353 "See the man page lftp(1) for a complete documentation.\n"
2354 )
2355 );
2356 }
2357