1// Package cmd implements the rclone command 2// 3// It is in a sub package so it's internals can be re-used elsewhere 4package cmd 5 6// FIXME only attach the remote flags when using a remote??? 7// would probably mean bringing all the flags in to here? Or define some flagsets in fs... 8 9import ( 10 "context" 11 "fmt" 12 "log" 13 "os" 14 "os/exec" 15 "path" 16 "regexp" 17 "runtime" 18 "runtime/pprof" 19 "strconv" 20 "strings" 21 "sync" 22 "time" 23 24 "github.com/pkg/errors" 25 "github.com/rclone/rclone/fs" 26 "github.com/rclone/rclone/fs/accounting" 27 "github.com/rclone/rclone/fs/cache" 28 "github.com/rclone/rclone/fs/config/configfile" 29 "github.com/rclone/rclone/fs/config/configflags" 30 "github.com/rclone/rclone/fs/config/flags" 31 "github.com/rclone/rclone/fs/filter" 32 "github.com/rclone/rclone/fs/filter/filterflags" 33 "github.com/rclone/rclone/fs/fserrors" 34 "github.com/rclone/rclone/fs/fspath" 35 fslog "github.com/rclone/rclone/fs/log" 36 "github.com/rclone/rclone/fs/rc/rcflags" 37 "github.com/rclone/rclone/fs/rc/rcserver" 38 "github.com/rclone/rclone/lib/atexit" 39 "github.com/rclone/rclone/lib/buildinfo" 40 "github.com/rclone/rclone/lib/exitcode" 41 "github.com/rclone/rclone/lib/random" 42 "github.com/rclone/rclone/lib/terminal" 43 "github.com/spf13/cobra" 44 "github.com/spf13/pflag" 45) 46 47// Globals 48var ( 49 // Flags 50 cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file") 51 memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file") 52 statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)") 53 dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second") 54 version bool 55 retries = flags.IntP("retries", "", 3, "Retry operations this many times if they fail") 56 retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)") 57 // Errors 58 errorCommandNotFound = errors.New("command not found") 59 errorUncategorized = errors.New("uncategorized error") 60 errorNotEnoughArguments = errors.New("not enough arguments") 61 errorTooManyArguments = errors.New("too many arguments") 62) 63 64// ShowVersion prints the version to stdout 65func ShowVersion() { 66 osVersion, osKernel := buildinfo.GetOSVersion() 67 if osVersion == "" { 68 osVersion = "unknown" 69 } 70 if osKernel == "" { 71 osKernel = "unknown" 72 } 73 74 linking, tagString := buildinfo.GetLinkingAndTags() 75 76 fmt.Printf("rclone %s\n", fs.Version) 77 fmt.Printf("- os/version: %s\n", osVersion) 78 fmt.Printf("- os/kernel: %s\n", osKernel) 79 fmt.Printf("- os/type: %s\n", runtime.GOOS) 80 fmt.Printf("- os/arch: %s\n", runtime.GOARCH) 81 fmt.Printf("- go/version: %s\n", runtime.Version()) 82 fmt.Printf("- go/linking: %s\n", linking) 83 fmt.Printf("- go/tags: %s\n", tagString) 84} 85 86// NewFsFile creates an Fs from a name but may point to a file. 87// 88// It returns a string with the file name if points to a file 89// otherwise "". 90func NewFsFile(remote string) (fs.Fs, string) { 91 _, fsPath, err := fspath.SplitFs(remote) 92 if err != nil { 93 err = fs.CountError(err) 94 log.Fatalf("Failed to create file system for %q: %v", remote, err) 95 } 96 f, err := cache.Get(context.Background(), remote) 97 switch err { 98 case fs.ErrorIsFile: 99 cache.Pin(f) // pin indefinitely since it was on the CLI 100 return f, path.Base(fsPath) 101 case nil: 102 cache.Pin(f) // pin indefinitely since it was on the CLI 103 return f, "" 104 default: 105 err = fs.CountError(err) 106 log.Fatalf("Failed to create file system for %q: %v", remote, err) 107 } 108 return nil, "" 109} 110 111// newFsFileAddFilter creates an src Fs from a name 112// 113// This works the same as NewFsFile however it adds filters to the Fs 114// to limit it to a single file if the remote pointed to a file. 115func newFsFileAddFilter(remote string) (fs.Fs, string) { 116 fi := filter.GetConfig(context.Background()) 117 f, fileName := NewFsFile(remote) 118 if fileName != "" { 119 if !fi.InActive() { 120 err := errors.Errorf("Can't limit to single files when using filters: %v", remote) 121 err = fs.CountError(err) 122 log.Fatalf(err.Error()) 123 } 124 // Limit transfers to this file 125 err := fi.AddFile(fileName) 126 if err != nil { 127 err = fs.CountError(err) 128 log.Fatalf("Failed to limit to single file %q: %v", remote, err) 129 } 130 } 131 return f, fileName 132} 133 134// NewFsSrc creates a new src fs from the arguments. 135// 136// The source can be a file or a directory - if a file then it will 137// limit the Fs to a single file. 138func NewFsSrc(args []string) fs.Fs { 139 fsrc, _ := newFsFileAddFilter(args[0]) 140 return fsrc 141} 142 143// newFsDir creates an Fs from a name 144// 145// This must point to a directory 146func newFsDir(remote string) fs.Fs { 147 f, err := cache.Get(context.Background(), remote) 148 if err != nil { 149 err = fs.CountError(err) 150 log.Fatalf("Failed to create file system for %q: %v", remote, err) 151 } 152 cache.Pin(f) // pin indefinitely since it was on the CLI 153 return f 154} 155 156// NewFsDir creates a new Fs from the arguments 157// 158// The argument must point a directory 159func NewFsDir(args []string) fs.Fs { 160 fdst := newFsDir(args[0]) 161 return fdst 162} 163 164// NewFsSrcDst creates a new src and dst fs from the arguments 165func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) { 166 fsrc, _ := newFsFileAddFilter(args[0]) 167 fdst := newFsDir(args[1]) 168 return fsrc, fdst 169} 170 171// NewFsSrcFileDst creates a new src and dst fs from the arguments 172// 173// The source may be a file, in which case the source Fs and file name is returned 174func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) { 175 fsrc, srcFileName = NewFsFile(args[0]) 176 fdst = newFsDir(args[1]) 177 return fsrc, srcFileName, fdst 178} 179 180// NewFsSrcDstFiles creates a new src and dst fs from the arguments 181// If src is a file then srcFileName and dstFileName will be non-empty 182func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) { 183 fsrc, srcFileName = newFsFileAddFilter(args[0]) 184 // If copying a file... 185 dstRemote := args[1] 186 // If file exists then srcFileName != "", however if the file 187 // doesn't exist then we assume it is a directory... 188 if srcFileName != "" { 189 var err error 190 dstRemote, dstFileName, err = fspath.Split(dstRemote) 191 if err != nil { 192 log.Fatalf("Parsing %q failed: %v", args[1], err) 193 } 194 if dstRemote == "" { 195 dstRemote = "." 196 } 197 if dstFileName == "" { 198 log.Fatalf("%q is a directory", args[1]) 199 } 200 } 201 fdst, err := cache.Get(context.Background(), dstRemote) 202 switch err { 203 case fs.ErrorIsFile: 204 _ = fs.CountError(err) 205 log.Fatalf("Source doesn't exist or is a directory and destination is a file") 206 case nil: 207 default: 208 _ = fs.CountError(err) 209 log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err) 210 } 211 cache.Pin(fdst) // pin indefinitely since it was on the CLI 212 return 213} 214 215// NewFsDstFile creates a new dst fs with a destination file name from the arguments 216func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) { 217 dstRemote, dstFileName, err := fspath.Split(args[0]) 218 if err != nil { 219 log.Fatalf("Parsing %q failed: %v", args[0], err) 220 } 221 if dstRemote == "" { 222 dstRemote = "." 223 } 224 if dstFileName == "" { 225 log.Fatalf("%q is a directory", args[0]) 226 } 227 fdst = newFsDir(dstRemote) 228 return 229} 230 231// ShowStats returns true if the user added a `--stats` flag to the command line. 232// 233// This is called by Run to override the default value of the 234// showStats passed in. 235func ShowStats() bool { 236 statsIntervalFlag := pflag.Lookup("stats") 237 return statsIntervalFlag != nil && statsIntervalFlag.Changed 238} 239 240// Run the function with stats and retries if required 241func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { 242 ci := fs.GetConfig(context.Background()) 243 var cmdErr error 244 stopStats := func() {} 245 if !showStats && ShowStats() { 246 showStats = true 247 } 248 if ci.Progress { 249 stopStats = startProgress() 250 } else if showStats { 251 stopStats = StartStats() 252 } 253 SigInfoHandler() 254 for try := 1; try <= *retries; try++ { 255 cmdErr = f() 256 cmdErr = fs.CountError(cmdErr) 257 lastErr := accounting.GlobalStats().GetLastError() 258 if cmdErr == nil { 259 cmdErr = lastErr 260 } 261 if !Retry || !accounting.GlobalStats().Errored() { 262 if try > 1 { 263 fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries) 264 } 265 break 266 } 267 if accounting.GlobalStats().HadFatalError() { 268 fs.Errorf(nil, "Fatal error received - not attempting retries") 269 break 270 } 271 if accounting.GlobalStats().Errored() && !accounting.GlobalStats().HadRetryError() { 272 fs.Errorf(nil, "Can't retry any of the errors - not attempting retries") 273 break 274 } 275 if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() { 276 d := retryAfter.Sub(time.Now()) 277 if d > 0 { 278 fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) 279 time.Sleep(d) 280 } 281 } 282 if lastErr != nil { 283 fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.GlobalStats().GetErrors(), lastErr) 284 } else { 285 fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.GlobalStats().GetErrors()) 286 } 287 if try < *retries { 288 accounting.GlobalStats().ResetErrors() 289 } 290 if *retriesInterval > 0 { 291 time.Sleep(*retriesInterval) 292 } 293 } 294 stopStats() 295 if showStats && (accounting.GlobalStats().Errored() || *statsInterval > 0) { 296 accounting.GlobalStats().Log() 297 } 298 fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine()) 299 300 if ci.Progress && ci.ProgressTerminalTitle { 301 // Clear terminal title 302 terminal.WriteTerminalTitle("") 303 } 304 305 // dump all running go-routines 306 if ci.Dump&fs.DumpGoRoutines != 0 { 307 err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 308 if err != nil { 309 fs.Errorf(nil, "Failed to dump goroutines: %v", err) 310 } 311 } 312 313 // dump open files 314 if ci.Dump&fs.DumpOpenFiles != 0 { 315 c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid())) 316 c.Stdout = os.Stdout 317 c.Stderr = os.Stderr 318 err := c.Run() 319 if err != nil { 320 fs.Errorf(nil, "Failed to list open files: %v", err) 321 } 322 } 323 324 // Log the final error message and exit 325 if cmdErr != nil { 326 nerrs := accounting.GlobalStats().GetErrors() 327 if nerrs <= 1 { 328 log.Printf("Failed to %s: %v", cmd.Name(), cmdErr) 329 } else { 330 log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, cmdErr) 331 } 332 } 333 resolveExitCode(cmdErr) 334 335} 336 337// CheckArgs checks there are enough arguments and prints a message if not 338func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) { 339 if len(args) < MinArgs { 340 _ = cmd.Usage() 341 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args) 342 resolveExitCode(errorNotEnoughArguments) 343 } else if len(args) > MaxArgs { 344 _ = cmd.Usage() 345 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args) 346 resolveExitCode(errorTooManyArguments) 347 } 348} 349 350// StartStats prints the stats every statsInterval 351// 352// It returns a func which should be called to stop the stats. 353func StartStats() func() { 354 if *statsInterval <= 0 { 355 return func() {} 356 } 357 stopStats := make(chan struct{}) 358 var wg sync.WaitGroup 359 wg.Add(1) 360 go func() { 361 defer wg.Done() 362 ticker := time.NewTicker(*statsInterval) 363 for { 364 select { 365 case <-ticker.C: 366 accounting.GlobalStats().Log() 367 case <-stopStats: 368 ticker.Stop() 369 return 370 } 371 } 372 }() 373 return func() { 374 close(stopStats) 375 wg.Wait() 376 } 377} 378 379// initConfig is run by cobra after initialising the flags 380func initConfig() { 381 ctx := context.Background() 382 ci := fs.GetConfig(ctx) 383 384 // Start the logger 385 fslog.InitLogging() 386 387 // Finish parsing any command line flags 388 configflags.SetFlags(ci) 389 390 // Load the config 391 configfile.Install() 392 393 // Start accounting 394 accounting.Start(ctx) 395 396 // Hide console window 397 if ci.NoConsole { 398 terminal.HideConsole() 399 } 400 401 // Load filters 402 err := filterflags.Reload(ctx) 403 if err != nil { 404 log.Fatalf("Failed to load filters: %v", err) 405 } 406 407 // Write the args for debug purposes 408 fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) 409 410 // Inform user about systemd log support now that we have a logger 411 if fslog.Opt.LogSystemdSupport { 412 fs.Debugf("rclone", "systemd logging support activated") 413 } 414 415 // Start the remote control server if configured 416 _, err = rcserver.Start(context.Background(), &rcflags.Opt) 417 if err != nil { 418 log.Fatalf("Failed to start remote control: %v", err) 419 } 420 421 // Setup CPU profiling if desired 422 if *cpuProfile != "" { 423 fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) 424 f, err := os.Create(*cpuProfile) 425 if err != nil { 426 err = fs.CountError(err) 427 log.Fatal(err) 428 } 429 err = pprof.StartCPUProfile(f) 430 if err != nil { 431 err = fs.CountError(err) 432 log.Fatal(err) 433 } 434 atexit.Register(func() { 435 pprof.StopCPUProfile() 436 }) 437 } 438 439 // Setup memory profiling if desired 440 if *memProfile != "" { 441 atexit.Register(func() { 442 fs.Infof(nil, "Saving Memory profile %q\n", *memProfile) 443 f, err := os.Create(*memProfile) 444 if err != nil { 445 err = fs.CountError(err) 446 log.Fatal(err) 447 } 448 err = pprof.WriteHeapProfile(f) 449 if err != nil { 450 err = fs.CountError(err) 451 log.Fatal(err) 452 } 453 err = f.Close() 454 if err != nil { 455 err = fs.CountError(err) 456 log.Fatal(err) 457 } 458 }) 459 } 460 461 if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false { 462 fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") 463 ci.DataRateUnit = "bytes" 464 } else { 465 ci.DataRateUnit = *dataRateUnit 466 } 467} 468 469func resolveExitCode(err error) { 470 ci := fs.GetConfig(context.Background()) 471 atexit.Run() 472 if err == nil { 473 if ci.ErrorOnNoTransfer { 474 if accounting.GlobalStats().GetTransfers() == 0 { 475 os.Exit(exitcode.NoFilesTransferred) 476 } 477 } 478 os.Exit(exitcode.Success) 479 } 480 481 _, unwrapped := fserrors.Cause(err) 482 483 switch { 484 case unwrapped == fs.ErrorDirNotFound: 485 os.Exit(exitcode.DirNotFound) 486 case unwrapped == fs.ErrorObjectNotFound: 487 os.Exit(exitcode.FileNotFound) 488 case unwrapped == errorUncategorized: 489 os.Exit(exitcode.UncategorizedError) 490 case unwrapped == accounting.ErrorMaxTransferLimitReached: 491 os.Exit(exitcode.TransferExceeded) 492 case fserrors.ShouldRetry(err): 493 os.Exit(exitcode.RetryError) 494 case fserrors.IsNoRetryError(err): 495 os.Exit(exitcode.NoRetryError) 496 case fserrors.IsFatalError(err): 497 os.Exit(exitcode.FatalError) 498 default: 499 os.Exit(exitcode.UsageError) 500 } 501} 502 503var backendFlags map[string]struct{} 504 505// AddBackendFlags creates flags for all the backend options 506func AddBackendFlags() { 507 backendFlags = map[string]struct{}{} 508 for _, fsInfo := range fs.Registry { 509 done := map[string]struct{}{} 510 for i := range fsInfo.Options { 511 opt := &fsInfo.Options[i] 512 // Skip if done already (e.g. with Provider options) 513 if _, doneAlready := done[opt.Name]; doneAlready { 514 continue 515 } 516 done[opt.Name] = struct{}{} 517 // Make a flag from each option 518 name := opt.FlagName(fsInfo.Prefix) 519 found := pflag.CommandLine.Lookup(name) != nil 520 if !found { 521 // Take first line of help only 522 help := strings.TrimSpace(opt.Help) 523 if nl := strings.IndexRune(help, '\n'); nl >= 0 { 524 help = help[:nl] 525 } 526 help = strings.TrimRight(strings.TrimSpace(help), ".!?") 527 if opt.IsPassword { 528 help += " (obscured)" 529 } 530 flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help) 531 flags.SetDefaultFromEnv(pflag.CommandLine, name) 532 if _, isBool := opt.Default.(bool); isBool { 533 flag.NoOptDefVal = "true" 534 } 535 // Hide on the command line if requested 536 if opt.Hide&fs.OptionHideCommandLine != 0 { 537 flag.Hidden = true 538 } 539 backendFlags[name] = struct{}{} 540 } else { 541 fs.Errorf(nil, "Not adding duplicate flag --%s", name) 542 } 543 //flag.Hidden = true 544 } 545 } 546} 547 548// Main runs rclone interpreting flags and commands out of os.Args 549func Main() { 550 if err := random.Seed(); err != nil { 551 log.Fatalf("Fatal error: %v", err) 552 } 553 setupRootCommand(Root) 554 AddBackendFlags() 555 if err := Root.Execute(); err != nil { 556 if strings.HasPrefix(err.Error(), "unknown command") && selfupdateEnabled { 557 Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath()) 558 } 559 log.Fatalf("Fatal error: %v", err) 560 } 561} 562