1// Test sync/copy/move 2 3package sync 4 5import ( 6 "context" 7 "fmt" 8 "runtime" 9 "strings" 10 "testing" 11 "time" 12 13 "github.com/pkg/errors" 14 _ "github.com/rclone/rclone/backend/all" // import all backends 15 "github.com/rclone/rclone/fs" 16 "github.com/rclone/rclone/fs/accounting" 17 "github.com/rclone/rclone/fs/filter" 18 "github.com/rclone/rclone/fs/fserrors" 19 "github.com/rclone/rclone/fs/hash" 20 "github.com/rclone/rclone/fs/operations" 21 "github.com/rclone/rclone/fstest" 22 "github.com/stretchr/testify/assert" 23 "github.com/stretchr/testify/require" 24 "golang.org/x/text/unicode/norm" 25) 26 27// Some times used in the tests 28var ( 29 t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") 30 t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") 31 t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") 32) 33 34// TestMain drives the tests 35func TestMain(m *testing.M) { 36 fstest.TestMain(m) 37} 38 39// Check dry run is working 40func TestCopyWithDryRun(t *testing.T) { 41 ctx := context.Background() 42 ctx, ci := fs.AddConfig(ctx) 43 r := fstest.NewRun(t) 44 defer r.Finalise() 45 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 46 r.Mkdir(ctx, r.Fremote) 47 48 ci.DryRun = true 49 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 50 require.NoError(t, err) 51 52 fstest.CheckItems(t, r.Flocal, file1) 53 fstest.CheckItems(t, r.Fremote) 54} 55 56// Now without dry run 57func TestCopy(t *testing.T) { 58 ctx := context.Background() 59 r := fstest.NewRun(t) 60 defer r.Finalise() 61 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 62 r.Mkdir(ctx, r.Fremote) 63 64 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 65 require.NoError(t, err) 66 67 fstest.CheckItems(t, r.Flocal, file1) 68 fstest.CheckItems(t, r.Fremote, file1) 69} 70 71func TestCopyMissingDirectory(t *testing.T) { 72 ctx := context.Background() 73 r := fstest.NewRun(t) 74 defer r.Finalise() 75 r.Mkdir(ctx, r.Fremote) 76 77 nonExistingFs, err := fs.NewFs(ctx, "/non-existing") 78 if err != nil { 79 t.Fatal(err) 80 } 81 82 err = CopyDir(ctx, r.Fremote, nonExistingFs, false) 83 require.Error(t, err) 84} 85 86// Now with --no-traverse 87func TestCopyNoTraverse(t *testing.T) { 88 ctx := context.Background() 89 ctx, ci := fs.AddConfig(ctx) 90 r := fstest.NewRun(t) 91 defer r.Finalise() 92 93 ci.NoTraverse = true 94 95 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 96 97 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 98 require.NoError(t, err) 99 100 fstest.CheckItems(t, r.Flocal, file1) 101 fstest.CheckItems(t, r.Fremote, file1) 102} 103 104// Now with --check-first 105func TestCopyCheckFirst(t *testing.T) { 106 ctx := context.Background() 107 ctx, ci := fs.AddConfig(ctx) 108 r := fstest.NewRun(t) 109 defer r.Finalise() 110 111 ci.CheckFirst = true 112 113 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 114 115 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 116 require.NoError(t, err) 117 118 fstest.CheckItems(t, r.Flocal, file1) 119 fstest.CheckItems(t, r.Fremote, file1) 120} 121 122// Now with --no-traverse 123func TestSyncNoTraverse(t *testing.T) { 124 ctx := context.Background() 125 ctx, ci := fs.AddConfig(ctx) 126 r := fstest.NewRun(t) 127 defer r.Finalise() 128 129 ci.NoTraverse = true 130 131 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 132 133 accounting.GlobalStats().ResetCounters() 134 err := Sync(ctx, r.Fremote, r.Flocal, false) 135 require.NoError(t, err) 136 137 fstest.CheckItems(t, r.Flocal, file1) 138 fstest.CheckItems(t, r.Fremote, file1) 139} 140 141// Test copy with depth 142func TestCopyWithDepth(t *testing.T) { 143 ctx := context.Background() 144 ctx, ci := fs.AddConfig(ctx) 145 r := fstest.NewRun(t) 146 defer r.Finalise() 147 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 148 file2 := r.WriteFile("hello world2", "hello world2", t2) 149 150 // Check the MaxDepth too 151 ci.MaxDepth = 1 152 153 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 154 require.NoError(t, err) 155 156 fstest.CheckItems(t, r.Flocal, file1, file2) 157 fstest.CheckItems(t, r.Fremote, file2) 158} 159 160// Test copy with files from 161func testCopyWithFilesFrom(t *testing.T, noTraverse bool) { 162 ctx := context.Background() 163 ctx, ci := fs.AddConfig(ctx) 164 r := fstest.NewRun(t) 165 defer r.Finalise() 166 file1 := r.WriteFile("potato2", "hello world", t1) 167 file2 := r.WriteFile("hello world2", "hello world2", t2) 168 169 // Set the --files-from equivalent 170 f, err := filter.NewFilter(nil) 171 require.NoError(t, err) 172 require.NoError(t, f.AddFile("potato2")) 173 require.NoError(t, f.AddFile("notfound")) 174 175 // Change the active filter 176 ctx = filter.ReplaceConfig(ctx, f) 177 178 ci.NoTraverse = noTraverse 179 180 err = CopyDir(ctx, r.Fremote, r.Flocal, false) 181 require.NoError(t, err) 182 183 fstest.CheckItems(t, r.Flocal, file1, file2) 184 fstest.CheckItems(t, r.Fremote, file1) 185} 186func TestCopyWithFilesFrom(t *testing.T) { testCopyWithFilesFrom(t, false) } 187func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t, true) } 188 189// Test copy empty directories 190func TestCopyEmptyDirectories(t *testing.T) { 191 ctx := context.Background() 192 r := fstest.NewRun(t) 193 defer r.Finalise() 194 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 195 err := operations.Mkdir(ctx, r.Flocal, "sub dir2") 196 require.NoError(t, err) 197 r.Mkdir(ctx, r.Fremote) 198 199 err = CopyDir(ctx, r.Fremote, r.Flocal, true) 200 require.NoError(t, err) 201 202 fstest.CheckListingWithPrecision( 203 t, 204 r.Fremote, 205 []fstest.Item{ 206 file1, 207 }, 208 []string{ 209 "sub dir", 210 "sub dir2", 211 }, 212 fs.GetModifyWindow(ctx, r.Fremote), 213 ) 214} 215 216// Test move empty directories 217func TestMoveEmptyDirectories(t *testing.T) { 218 ctx := context.Background() 219 r := fstest.NewRun(t) 220 defer r.Finalise() 221 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 222 err := operations.Mkdir(ctx, r.Flocal, "sub dir2") 223 require.NoError(t, err) 224 r.Mkdir(ctx, r.Fremote) 225 226 err = MoveDir(ctx, r.Fremote, r.Flocal, false, true) 227 require.NoError(t, err) 228 229 fstest.CheckListingWithPrecision( 230 t, 231 r.Fremote, 232 []fstest.Item{ 233 file1, 234 }, 235 []string{ 236 "sub dir", 237 "sub dir2", 238 }, 239 fs.GetModifyWindow(ctx, r.Fremote), 240 ) 241} 242 243// Test sync empty directories 244func TestSyncEmptyDirectories(t *testing.T) { 245 ctx := context.Background() 246 r := fstest.NewRun(t) 247 defer r.Finalise() 248 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 249 err := operations.Mkdir(ctx, r.Flocal, "sub dir2") 250 require.NoError(t, err) 251 r.Mkdir(ctx, r.Fremote) 252 253 err = Sync(ctx, r.Fremote, r.Flocal, true) 254 require.NoError(t, err) 255 256 fstest.CheckListingWithPrecision( 257 t, 258 r.Fremote, 259 []fstest.Item{ 260 file1, 261 }, 262 []string{ 263 "sub dir", 264 "sub dir2", 265 }, 266 fs.GetModifyWindow(ctx, r.Fremote), 267 ) 268} 269 270// Test a server-side copy if possible, or the backup path if not 271func TestServerSideCopy(t *testing.T) { 272 ctx := context.Background() 273 r := fstest.NewRun(t) 274 defer r.Finalise() 275 file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) 276 fstest.CheckItems(t, r.Fremote, file1) 277 278 FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote() 279 require.NoError(t, err) 280 defer finaliseCopy() 281 t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) 282 283 err = CopyDir(ctx, FremoteCopy, r.Fremote, false) 284 require.NoError(t, err) 285 286 fstest.CheckItems(t, FremoteCopy, file1) 287} 288 289// Check that if the local file doesn't exist when we copy it up, 290// nothing happens to the remote file 291func TestCopyAfterDelete(t *testing.T) { 292 ctx := context.Background() 293 r := fstest.NewRun(t) 294 defer r.Finalise() 295 file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) 296 fstest.CheckItems(t, r.Flocal) 297 fstest.CheckItems(t, r.Fremote, file1) 298 299 err := operations.Mkdir(ctx, r.Flocal, "") 300 require.NoError(t, err) 301 302 err = CopyDir(ctx, r.Fremote, r.Flocal, false) 303 require.NoError(t, err) 304 305 fstest.CheckItems(t, r.Flocal) 306 fstest.CheckItems(t, r.Fremote, file1) 307} 308 309// Check the copy downloading a file 310func TestCopyRedownload(t *testing.T) { 311 ctx := context.Background() 312 r := fstest.NewRun(t) 313 defer r.Finalise() 314 file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) 315 fstest.CheckItems(t, r.Fremote, file1) 316 317 err := CopyDir(ctx, r.Flocal, r.Fremote, false) 318 require.NoError(t, err) 319 320 // Test with combined precision of local and remote as we copied it there and back 321 fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1}, nil, fs.GetModifyWindow(ctx, r.Flocal, r.Fremote)) 322} 323 324// Create a file and sync it. Change the last modified date and resync. 325// If we're only doing sync by size and checksum, we expect nothing to 326// to be transferred on the second sync. 327func TestSyncBasedOnCheckSum(t *testing.T) { 328 ctx := context.Background() 329 ctx, ci := fs.AddConfig(ctx) 330 r := fstest.NewRun(t) 331 defer r.Finalise() 332 ci.CheckSum = true 333 334 file1 := r.WriteFile("check sum", "-", t1) 335 fstest.CheckItems(t, r.Flocal, file1) 336 337 accounting.GlobalStats().ResetCounters() 338 err := Sync(ctx, r.Fremote, r.Flocal, false) 339 require.NoError(t, err) 340 341 // We should have transferred exactly one file. 342 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 343 fstest.CheckItems(t, r.Fremote, file1) 344 345 // Change last modified date only 346 file2 := r.WriteFile("check sum", "-", t2) 347 fstest.CheckItems(t, r.Flocal, file2) 348 349 accounting.GlobalStats().ResetCounters() 350 err = Sync(ctx, r.Fremote, r.Flocal, false) 351 require.NoError(t, err) 352 353 // We should have transferred no files 354 assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) 355 fstest.CheckItems(t, r.Flocal, file2) 356 fstest.CheckItems(t, r.Fremote, file1) 357} 358 359// Create a file and sync it. Change the last modified date and the 360// file contents but not the size. If we're only doing sync by size 361// only, we expect nothing to to be transferred on the second sync. 362func TestSyncSizeOnly(t *testing.T) { 363 ctx := context.Background() 364 ctx, ci := fs.AddConfig(ctx) 365 r := fstest.NewRun(t) 366 defer r.Finalise() 367 ci.SizeOnly = true 368 369 file1 := r.WriteFile("sizeonly", "potato", t1) 370 fstest.CheckItems(t, r.Flocal, file1) 371 372 accounting.GlobalStats().ResetCounters() 373 err := Sync(ctx, r.Fremote, r.Flocal, false) 374 require.NoError(t, err) 375 376 // We should have transferred exactly one file. 377 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 378 fstest.CheckItems(t, r.Fremote, file1) 379 380 // Update mtime, md5sum but not length of file 381 file2 := r.WriteFile("sizeonly", "POTATO", t2) 382 fstest.CheckItems(t, r.Flocal, file2) 383 384 accounting.GlobalStats().ResetCounters() 385 err = Sync(ctx, r.Fremote, r.Flocal, false) 386 require.NoError(t, err) 387 388 // We should have transferred no files 389 assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) 390 fstest.CheckItems(t, r.Flocal, file2) 391 fstest.CheckItems(t, r.Fremote, file1) 392} 393 394// Create a file and sync it. Keep the last modified date but change 395// the size. With --ignore-size we expect nothing to to be 396// transferred on the second sync. 397func TestSyncIgnoreSize(t *testing.T) { 398 ctx := context.Background() 399 ctx, ci := fs.AddConfig(ctx) 400 r := fstest.NewRun(t) 401 defer r.Finalise() 402 ci.IgnoreSize = true 403 404 file1 := r.WriteFile("ignore-size", "contents", t1) 405 fstest.CheckItems(t, r.Flocal, file1) 406 407 accounting.GlobalStats().ResetCounters() 408 err := Sync(ctx, r.Fremote, r.Flocal, false) 409 require.NoError(t, err) 410 411 // We should have transferred exactly one file. 412 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 413 fstest.CheckItems(t, r.Fremote, file1) 414 415 // Update size but not date of file 416 file2 := r.WriteFile("ignore-size", "longer contents but same date", t1) 417 fstest.CheckItems(t, r.Flocal, file2) 418 419 accounting.GlobalStats().ResetCounters() 420 err = Sync(ctx, r.Fremote, r.Flocal, false) 421 require.NoError(t, err) 422 423 // We should have transferred no files 424 assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) 425 fstest.CheckItems(t, r.Flocal, file2) 426 fstest.CheckItems(t, r.Fremote, file1) 427} 428 429func TestSyncIgnoreTimes(t *testing.T) { 430 ctx := context.Background() 431 ctx, ci := fs.AddConfig(ctx) 432 r := fstest.NewRun(t) 433 defer r.Finalise() 434 file1 := r.WriteBoth(ctx, "existing", "potato", t1) 435 fstest.CheckItems(t, r.Fremote, file1) 436 437 accounting.GlobalStats().ResetCounters() 438 err := Sync(ctx, r.Fremote, r.Flocal, false) 439 require.NoError(t, err) 440 441 // We should have transferred exactly 0 files because the 442 // files were identical. 443 assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) 444 445 ci.IgnoreTimes = true 446 447 accounting.GlobalStats().ResetCounters() 448 err = Sync(ctx, r.Fremote, r.Flocal, false) 449 require.NoError(t, err) 450 451 // We should have transferred exactly one file even though the 452 // files were identical. 453 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 454 455 fstest.CheckItems(t, r.Flocal, file1) 456 fstest.CheckItems(t, r.Fremote, file1) 457} 458 459func TestSyncIgnoreExisting(t *testing.T) { 460 ctx := context.Background() 461 ctx, ci := fs.AddConfig(ctx) 462 r := fstest.NewRun(t) 463 defer r.Finalise() 464 file1 := r.WriteFile("existing", "potato", t1) 465 466 ci.IgnoreExisting = true 467 468 accounting.GlobalStats().ResetCounters() 469 err := Sync(ctx, r.Fremote, r.Flocal, false) 470 require.NoError(t, err) 471 fstest.CheckItems(t, r.Flocal, file1) 472 fstest.CheckItems(t, r.Fremote, file1) 473 474 // Change everything 475 r.WriteFile("existing", "newpotatoes", t2) 476 accounting.GlobalStats().ResetCounters() 477 err = Sync(ctx, r.Fremote, r.Flocal, false) 478 require.NoError(t, err) 479 // Items should not change 480 fstest.CheckItems(t, r.Fremote, file1) 481} 482 483func TestSyncIgnoreErrors(t *testing.T) { 484 ctx := context.Background() 485 ctx, ci := fs.AddConfig(ctx) 486 r := fstest.NewRun(t) 487 ci.IgnoreErrors = true 488 defer r.Finalise() 489 file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) 490 file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) 491 file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) 492 require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) 493 494 fstest.CheckListingWithPrecision( 495 t, 496 r.Flocal, 497 []fstest.Item{ 498 file1, 499 file3, 500 }, 501 []string{ 502 "a", 503 "c", 504 }, 505 fs.GetModifyWindow(ctx, r.Fremote), 506 ) 507 fstest.CheckListingWithPrecision( 508 t, 509 r.Fremote, 510 []fstest.Item{ 511 file2, 512 file3, 513 }, 514 []string{ 515 "b", 516 "c", 517 "d", 518 }, 519 fs.GetModifyWindow(ctx, r.Fremote), 520 ) 521 522 accounting.GlobalStats().ResetCounters() 523 _ = fs.CountError(errors.New("boom")) 524 assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 525 526 fstest.CheckListingWithPrecision( 527 t, 528 r.Flocal, 529 []fstest.Item{ 530 file1, 531 file3, 532 }, 533 []string{ 534 "a", 535 "c", 536 }, 537 fs.GetModifyWindow(ctx, r.Fremote), 538 ) 539 fstest.CheckListingWithPrecision( 540 t, 541 r.Fremote, 542 []fstest.Item{ 543 file1, 544 file3, 545 }, 546 []string{ 547 "a", 548 "c", 549 }, 550 fs.GetModifyWindow(ctx, r.Fremote), 551 ) 552} 553 554func TestSyncAfterChangingModtimeOnly(t *testing.T) { 555 ctx := context.Background() 556 ctx, ci := fs.AddConfig(ctx) 557 r := fstest.NewRun(t) 558 defer r.Finalise() 559 file1 := r.WriteFile("empty space", "-", t2) 560 file2 := r.WriteObject(ctx, "empty space", "-", t1) 561 562 fstest.CheckItems(t, r.Flocal, file1) 563 fstest.CheckItems(t, r.Fremote, file2) 564 565 ci.DryRun = true 566 567 accounting.GlobalStats().ResetCounters() 568 err := Sync(ctx, r.Fremote, r.Flocal, false) 569 require.NoError(t, err) 570 571 fstest.CheckItems(t, r.Flocal, file1) 572 fstest.CheckItems(t, r.Fremote, file2) 573 574 ci.DryRun = false 575 576 accounting.GlobalStats().ResetCounters() 577 err = Sync(ctx, r.Fremote, r.Flocal, false) 578 require.NoError(t, err) 579 580 fstest.CheckItems(t, r.Flocal, file1) 581 fstest.CheckItems(t, r.Fremote, file1) 582} 583 584func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) { 585 ctx := context.Background() 586 ctx, ci := fs.AddConfig(ctx) 587 r := fstest.NewRun(t) 588 defer r.Finalise() 589 590 if r.Fremote.Hashes().Count() == 0 { 591 t.Logf("Can't check this if no hashes supported") 592 return 593 } 594 595 ci.NoUpdateModTime = true 596 597 file1 := r.WriteFile("empty space", "-", t2) 598 file2 := r.WriteObject(ctx, "empty space", "-", t1) 599 600 fstest.CheckItems(t, r.Flocal, file1) 601 fstest.CheckItems(t, r.Fremote, file2) 602 603 accounting.GlobalStats().ResetCounters() 604 err := Sync(ctx, r.Fremote, r.Flocal, false) 605 require.NoError(t, err) 606 607 fstest.CheckItems(t, r.Flocal, file1) 608 fstest.CheckItems(t, r.Fremote, file2) 609} 610 611func TestSyncDoesntUpdateModtime(t *testing.T) { 612 ctx := context.Background() 613 r := fstest.NewRun(t) 614 defer r.Finalise() 615 if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported { 616 t.Skip("Can't run this test on fs which doesn't support mod time") 617 } 618 619 file1 := r.WriteFile("foo", "foo", t2) 620 file2 := r.WriteObject(ctx, "foo", "bar", t1) 621 622 fstest.CheckItems(t, r.Flocal, file1) 623 fstest.CheckItems(t, r.Fremote, file2) 624 625 accounting.GlobalStats().ResetCounters() 626 err := Sync(ctx, r.Fremote, r.Flocal, false) 627 require.NoError(t, err) 628 629 fstest.CheckItems(t, r.Flocal, file1) 630 fstest.CheckItems(t, r.Fremote, file1) 631 632 // We should have transferred exactly one file, not set the mod time 633 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 634} 635 636func TestSyncAfterAddingAFile(t *testing.T) { 637 ctx := context.Background() 638 r := fstest.NewRun(t) 639 defer r.Finalise() 640 file1 := r.WriteBoth(ctx, "empty space", "-", t2) 641 file2 := r.WriteFile("potato", "------------------------------------------------------------", t3) 642 643 fstest.CheckItems(t, r.Flocal, file1, file2) 644 fstest.CheckItems(t, r.Fremote, file1) 645 646 accounting.GlobalStats().ResetCounters() 647 err := Sync(ctx, r.Fremote, r.Flocal, false) 648 require.NoError(t, err) 649 fstest.CheckItems(t, r.Flocal, file1, file2) 650 fstest.CheckItems(t, r.Fremote, file1, file2) 651} 652 653func TestSyncAfterChangingFilesSizeOnly(t *testing.T) { 654 ctx := context.Background() 655 r := fstest.NewRun(t) 656 defer r.Finalise() 657 file1 := r.WriteObject(ctx, "potato", "------------------------------------------------------------", t3) 658 file2 := r.WriteFile("potato", "smaller but same date", t3) 659 fstest.CheckItems(t, r.Fremote, file1) 660 fstest.CheckItems(t, r.Flocal, file2) 661 662 accounting.GlobalStats().ResetCounters() 663 err := Sync(ctx, r.Fremote, r.Flocal, false) 664 require.NoError(t, err) 665 fstest.CheckItems(t, r.Flocal, file2) 666 fstest.CheckItems(t, r.Fremote, file2) 667} 668 669// Sync after changing a file's contents, changing modtime but length 670// remaining the same 671func TestSyncAfterChangingContentsOnly(t *testing.T) { 672 ctx := context.Background() 673 r := fstest.NewRun(t) 674 defer r.Finalise() 675 var file1 fstest.Item 676 if r.Fremote.Precision() == fs.ModTimeNotSupported { 677 t.Logf("ModTimeNotSupported so forcing file to be a different size") 678 file1 = r.WriteObject(ctx, "potato", "different size to make sure it syncs", t3) 679 } else { 680 file1 = r.WriteObject(ctx, "potato", "smaller but same date", t3) 681 } 682 file2 := r.WriteFile("potato", "SMALLER BUT SAME DATE", t2) 683 fstest.CheckItems(t, r.Fremote, file1) 684 fstest.CheckItems(t, r.Flocal, file2) 685 686 accounting.GlobalStats().ResetCounters() 687 err := Sync(ctx, r.Fremote, r.Flocal, false) 688 require.NoError(t, err) 689 fstest.CheckItems(t, r.Flocal, file2) 690 fstest.CheckItems(t, r.Fremote, file2) 691} 692 693// Sync after removing a file and adding a file --dry-run 694func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) { 695 ctx := context.Background() 696 ctx, ci := fs.AddConfig(ctx) 697 r := fstest.NewRun(t) 698 defer r.Finalise() 699 file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) 700 file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2) 701 file3 := r.WriteBoth(ctx, "empty space", "-", t2) 702 703 ci.DryRun = true 704 accounting.GlobalStats().ResetCounters() 705 err := Sync(ctx, r.Fremote, r.Flocal, false) 706 ci.DryRun = false 707 require.NoError(t, err) 708 709 fstest.CheckItems(t, r.Flocal, file3, file1) 710 fstest.CheckItems(t, r.Fremote, file3, file2) 711} 712 713// Sync after removing a file and adding a file 714func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T) { 715 r := fstest.NewRun(t) 716 defer r.Finalise() 717 file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) 718 file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2) 719 file3 := r.WriteBoth(ctx, "empty space", "-", t2) 720 fstest.CheckItems(t, r.Fremote, file2, file3) 721 fstest.CheckItems(t, r.Flocal, file1, file3) 722 723 accounting.GlobalStats().ResetCounters() 724 err := Sync(ctx, r.Fremote, r.Flocal, false) 725 require.NoError(t, err) 726 fstest.CheckItems(t, r.Flocal, file1, file3) 727 fstest.CheckItems(t, r.Fremote, file1, file3) 728} 729 730func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) { 731 testSyncAfterRemovingAFileAndAddingAFile(context.Background(), t) 732} 733 734// Sync after removing a file and adding a file 735func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *testing.T) { 736 r := fstest.NewRun(t) 737 defer r.Finalise() 738 file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) 739 file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) 740 file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) 741 require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) 742 require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d/e")) 743 744 fstest.CheckListingWithPrecision( 745 t, 746 r.Flocal, 747 []fstest.Item{ 748 file1, 749 file3, 750 }, 751 []string{ 752 "a", 753 "c", 754 }, 755 fs.GetModifyWindow(ctx, r.Fremote), 756 ) 757 fstest.CheckListingWithPrecision( 758 t, 759 r.Fremote, 760 []fstest.Item{ 761 file2, 762 file3, 763 }, 764 []string{ 765 "b", 766 "c", 767 "d", 768 "d/e", 769 }, 770 fs.GetModifyWindow(ctx, r.Fremote), 771 ) 772 773 accounting.GlobalStats().ResetCounters() 774 err := Sync(ctx, r.Fremote, r.Flocal, false) 775 require.NoError(t, err) 776 777 fstest.CheckListingWithPrecision( 778 t, 779 r.Flocal, 780 []fstest.Item{ 781 file1, 782 file3, 783 }, 784 []string{ 785 "a", 786 "c", 787 }, 788 fs.GetModifyWindow(ctx, r.Fremote), 789 ) 790 fstest.CheckListingWithPrecision( 791 t, 792 r.Fremote, 793 []fstest.Item{ 794 file1, 795 file3, 796 }, 797 []string{ 798 "a", 799 "c", 800 }, 801 fs.GetModifyWindow(ctx, r.Fremote), 802 ) 803} 804 805func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { 806 testSyncAfterRemovingAFileAndAddingAFileSubDir(context.Background(), t) 807} 808 809// Sync after removing a file and adding a file with IO Errors 810func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { 811 ctx := context.Background() 812 r := fstest.NewRun(t) 813 defer r.Finalise() 814 file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) 815 file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) 816 file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) 817 require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) 818 819 fstest.CheckListingWithPrecision( 820 t, 821 r.Flocal, 822 []fstest.Item{ 823 file1, 824 file3, 825 }, 826 []string{ 827 "a", 828 "c", 829 }, 830 fs.GetModifyWindow(ctx, r.Fremote), 831 ) 832 fstest.CheckListingWithPrecision( 833 t, 834 r.Fremote, 835 []fstest.Item{ 836 file2, 837 file3, 838 }, 839 []string{ 840 "b", 841 "c", 842 "d", 843 }, 844 fs.GetModifyWindow(ctx, r.Fremote), 845 ) 846 847 accounting.GlobalStats().ResetCounters() 848 _ = fs.CountError(errors.New("boom")) 849 err := Sync(ctx, r.Fremote, r.Flocal, false) 850 assert.Equal(t, fs.ErrorNotDeleting, err) 851 852 fstest.CheckListingWithPrecision( 853 t, 854 r.Flocal, 855 []fstest.Item{ 856 file1, 857 file3, 858 }, 859 []string{ 860 "a", 861 "c", 862 }, 863 fs.GetModifyWindow(ctx, r.Fremote), 864 ) 865 fstest.CheckListingWithPrecision( 866 t, 867 r.Fremote, 868 []fstest.Item{ 869 file1, 870 file2, 871 file3, 872 }, 873 []string{ 874 "a", 875 "b", 876 "c", 877 "d", 878 }, 879 fs.GetModifyWindow(ctx, r.Fremote), 880 ) 881} 882 883// Sync test delete after 884func TestSyncDeleteAfter(t *testing.T) { 885 ctx := context.Background() 886 ci := fs.GetConfig(ctx) 887 // This is the default so we've checked this already 888 // check it is the default 889 require.Equal(t, ci.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after") 890} 891 892// Sync test delete during 893func TestSyncDeleteDuring(t *testing.T) { 894 ctx := context.Background() 895 ctx, ci := fs.AddConfig(ctx) 896 ci.DeleteMode = fs.DeleteModeDuring 897 898 testSyncAfterRemovingAFileAndAddingAFile(ctx, t) 899} 900 901// Sync test delete before 902func TestSyncDeleteBefore(t *testing.T) { 903 ctx := context.Background() 904 ctx, ci := fs.AddConfig(ctx) 905 ci.DeleteMode = fs.DeleteModeBefore 906 907 testSyncAfterRemovingAFileAndAddingAFile(ctx, t) 908} 909 910// Copy test delete before - shouldn't delete anything 911func TestCopyDeleteBefore(t *testing.T) { 912 ctx := context.Background() 913 ctx, ci := fs.AddConfig(ctx) 914 r := fstest.NewRun(t) 915 defer r.Finalise() 916 917 ci.DeleteMode = fs.DeleteModeBefore 918 919 file1 := r.WriteObject(ctx, "potato", "hopefully not deleted", t1) 920 file2 := r.WriteFile("potato2", "hopefully copied in", t1) 921 fstest.CheckItems(t, r.Fremote, file1) 922 fstest.CheckItems(t, r.Flocal, file2) 923 924 accounting.GlobalStats().ResetCounters() 925 err := CopyDir(ctx, r.Fremote, r.Flocal, false) 926 require.NoError(t, err) 927 928 fstest.CheckItems(t, r.Fremote, file1, file2) 929 fstest.CheckItems(t, r.Flocal, file2) 930} 931 932// Test with exclude 933func TestSyncWithExclude(t *testing.T) { 934 ctx := context.Background() 935 r := fstest.NewRun(t) 936 defer r.Finalise() 937 file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) 938 file2 := r.WriteBoth(ctx, "empty space", "-", t2) 939 file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes 940 fstest.CheckItems(t, r.Fremote, file1, file2) 941 fstest.CheckItems(t, r.Flocal, file1, file2, file3) 942 943 fi, err := filter.NewFilter(nil) 944 require.NoError(t, err) 945 fi.Opt.MaxSize = 40 946 ctx = filter.ReplaceConfig(ctx, fi) 947 948 accounting.GlobalStats().ResetCounters() 949 err = Sync(ctx, r.Fremote, r.Flocal, false) 950 require.NoError(t, err) 951 fstest.CheckItems(t, r.Fremote, file2, file1) 952 953 // Now sync the other way round and check enormous doesn't get 954 // deleted as it is excluded from the sync 955 accounting.GlobalStats().ResetCounters() 956 err = Sync(ctx, r.Flocal, r.Fremote, false) 957 require.NoError(t, err) 958 fstest.CheckItems(t, r.Flocal, file2, file1, file3) 959} 960 961// Test with exclude and delete excluded 962func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) { 963 ctx := context.Background() 964 r := fstest.NewRun(t) 965 defer r.Finalise() 966 file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) // 60 bytes 967 file2 := r.WriteBoth(ctx, "empty space", "-", t2) 968 file3 := r.WriteBoth(ctx, "enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes 969 fstest.CheckItems(t, r.Fremote, file1, file2, file3) 970 fstest.CheckItems(t, r.Flocal, file1, file2, file3) 971 972 fi, err := filter.NewFilter(nil) 973 require.NoError(t, err) 974 fi.Opt.MaxSize = 40 975 fi.Opt.DeleteExcluded = true 976 ctx = filter.ReplaceConfig(ctx, fi) 977 978 accounting.GlobalStats().ResetCounters() 979 err = Sync(ctx, r.Fremote, r.Flocal, false) 980 require.NoError(t, err) 981 fstest.CheckItems(t, r.Fremote, file2) 982 983 // Check sync the other way round to make sure enormous gets 984 // deleted even though it is excluded 985 accounting.GlobalStats().ResetCounters() 986 err = Sync(ctx, r.Flocal, r.Fremote, false) 987 require.NoError(t, err) 988 fstest.CheckItems(t, r.Flocal, file2) 989} 990 991// Test with UpdateOlder set 992func TestSyncWithUpdateOlder(t *testing.T) { 993 ctx := context.Background() 994 ctx, ci := fs.AddConfig(ctx) 995 r := fstest.NewRun(t) 996 defer r.Finalise() 997 if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported { 998 t.Skip("Can't run this test on fs which doesn't support mod time") 999 } 1000 t2plus := t2.Add(time.Second / 2) 1001 t2minus := t2.Add(time.Second / 2) 1002 oneF := r.WriteFile("one", "one", t1) 1003 twoF := r.WriteFile("two", "two", t3) 1004 threeF := r.WriteFile("three", "three", t2) 1005 fourF := r.WriteFile("four", "four", t2) 1006 fiveF := r.WriteFile("five", "five", t2) 1007 fstest.CheckItems(t, r.Flocal, oneF, twoF, threeF, fourF, fiveF) 1008 oneO := r.WriteObject(ctx, "one", "ONE", t2) 1009 twoO := r.WriteObject(ctx, "two", "TWO", t2) 1010 threeO := r.WriteObject(ctx, "three", "THREE", t2plus) 1011 fourO := r.WriteObject(ctx, "four", "FOURFOUR", t2minus) 1012 fstest.CheckItems(t, r.Fremote, oneO, twoO, threeO, fourO) 1013 1014 ci.UpdateOlder = true 1015 ci.ModifyWindow = fs.ModTimeNotSupported 1016 1017 err := Sync(ctx, r.Fremote, r.Flocal, false) 1018 require.NoError(t, err) 1019 fstest.CheckItems(t, r.Fremote, oneO, twoF, threeO, fourF, fiveF) 1020 1021 if r.Fremote.Hashes().Count() == 0 { 1022 t.Logf("Skip test with --checksum as no hashes supported") 1023 return 1024 } 1025 1026 // now enable checksum 1027 ci.CheckSum = true 1028 1029 err = Sync(ctx, r.Fremote, r.Flocal, false) 1030 require.NoError(t, err) 1031 fstest.CheckItems(t, r.Fremote, oneO, twoF, threeF, fourF, fiveF) 1032} 1033 1034// Test with a max transfer duration 1035func TestSyncWithMaxDuration(t *testing.T) { 1036 ctx := context.Background() 1037 ctx, ci := fs.AddConfig(ctx) 1038 if *fstest.RemoteName != "" { 1039 t.Skip("Skipping test on non local remote") 1040 } 1041 r := fstest.NewRun(t) 1042 defer r.Finalise() 1043 1044 maxDuration := 250 * time.Millisecond 1045 ci.MaxDuration = maxDuration 1046 bytesPerSecond := 300 1047 accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: fs.SizeSuffix(bytesPerSecond), Rx: fs.SizeSuffix(bytesPerSecond)}) 1048 ci.Transfers = 1 1049 defer accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: -1, Rx: -1}) 1050 1051 // 5 files of 60 bytes at 60 Byte/s 5 seconds 1052 testFiles := make([]fstest.Item, 5) 1053 for i := 0; i < len(testFiles); i++ { 1054 testFiles[i] = r.WriteFile(fmt.Sprintf("file%d", i), "------------------------------------------------------------", t1) 1055 } 1056 1057 fstest.CheckListing(t, r.Flocal, testFiles) 1058 1059 accounting.GlobalStats().ResetCounters() 1060 startTime := time.Now() 1061 err := Sync(ctx, r.Fremote, r.Flocal, false) 1062 require.Equal(t, context.DeadlineExceeded, errors.Cause(err)) 1063 1064 elapsed := time.Since(startTime) 1065 maxTransferTime := (time.Duration(len(testFiles)) * 60 * time.Second) / time.Duration(bytesPerSecond) 1066 1067 what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime) 1068 require.True(t, elapsed >= maxDuration, what) 1069 require.True(t, elapsed < 5*time.Second, what) 1070 // we must not have transferred all files during the session 1071 require.True(t, accounting.GlobalStats().GetTransfers() < int64(len(testFiles))) 1072} 1073 1074// Test with TrackRenames set 1075func TestSyncWithTrackRenames(t *testing.T) { 1076 ctx := context.Background() 1077 ctx, ci := fs.AddConfig(ctx) 1078 r := fstest.NewRun(t) 1079 defer r.Finalise() 1080 1081 ci.TrackRenames = true 1082 defer func() { 1083 ci.TrackRenames = false 1084 }() 1085 1086 haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != hash.None 1087 canTrackRenames := haveHash && operations.CanServerSideMove(r.Fremote) 1088 t.Logf("Can track renames: %v", canTrackRenames) 1089 1090 f1 := r.WriteFile("potato", "Potato Content", t1) 1091 f2 := r.WriteFile("yam", "Yam Content", t2) 1092 1093 accounting.GlobalStats().ResetCounters() 1094 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1095 1096 fstest.CheckItems(t, r.Fremote, f1, f2) 1097 fstest.CheckItems(t, r.Flocal, f1, f2) 1098 1099 // Now rename locally. 1100 f2 = r.RenameFile(f2, "yaml") 1101 1102 accounting.GlobalStats().ResetCounters() 1103 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1104 1105 fstest.CheckItems(t, r.Fremote, f1, f2) 1106 1107 // Check we renamed something if we should have 1108 if canTrackRenames { 1109 renames := accounting.GlobalStats().Renames(0) 1110 assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) 1111 } 1112} 1113 1114func TestParseRenamesStrategyModtime(t *testing.T) { 1115 for _, test := range []struct { 1116 in string 1117 want trackRenamesStrategy 1118 wantErr bool 1119 }{ 1120 {"", 0, false}, 1121 {"modtime", trackRenamesStrategyModtime, false}, 1122 {"hash", trackRenamesStrategyHash, false}, 1123 {"size", 0, false}, 1124 {"modtime,hash", trackRenamesStrategyModtime | trackRenamesStrategyHash, false}, 1125 {"hash,modtime,size", trackRenamesStrategyModtime | trackRenamesStrategyHash, false}, 1126 {"size,boom", 0, true}, 1127 } { 1128 got, err := parseTrackRenamesStrategy(test.in) 1129 assert.Equal(t, test.want, got, test.in) 1130 assert.Equal(t, test.wantErr, err != nil, test.in) 1131 } 1132} 1133 1134func TestRenamesStrategyModtime(t *testing.T) { 1135 both := trackRenamesStrategyHash | trackRenamesStrategyModtime 1136 hash := trackRenamesStrategyHash 1137 modTime := trackRenamesStrategyModtime 1138 1139 assert.True(t, both.hash()) 1140 assert.True(t, both.modTime()) 1141 assert.True(t, hash.hash()) 1142 assert.False(t, hash.modTime()) 1143 assert.False(t, modTime.hash()) 1144 assert.True(t, modTime.modTime()) 1145} 1146 1147func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) { 1148 ctx := context.Background() 1149 ctx, ci := fs.AddConfig(ctx) 1150 r := fstest.NewRun(t) 1151 defer r.Finalise() 1152 1153 ci.TrackRenames = true 1154 ci.TrackRenamesStrategy = "modtime" 1155 1156 canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported 1157 t.Logf("Can track renames: %v", canTrackRenames) 1158 1159 f1 := r.WriteFile("potato", "Potato Content", t1) 1160 f2 := r.WriteFile("yam", "Yam Content", t2) 1161 1162 accounting.GlobalStats().ResetCounters() 1163 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1164 1165 fstest.CheckItems(t, r.Fremote, f1, f2) 1166 fstest.CheckItems(t, r.Flocal, f1, f2) 1167 1168 // Now rename locally. 1169 f2 = r.RenameFile(f2, "yaml") 1170 1171 accounting.GlobalStats().ResetCounters() 1172 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1173 1174 fstest.CheckItems(t, r.Fremote, f1, f2) 1175 1176 // Check we renamed something if we should have 1177 if canTrackRenames { 1178 renames := accounting.GlobalStats().Renames(0) 1179 assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) 1180 } 1181} 1182 1183func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) { 1184 ctx := context.Background() 1185 ctx, ci := fs.AddConfig(ctx) 1186 r := fstest.NewRun(t) 1187 defer r.Finalise() 1188 1189 ci.TrackRenames = true 1190 ci.TrackRenamesStrategy = "leaf" 1191 1192 canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported 1193 t.Logf("Can track renames: %v", canTrackRenames) 1194 1195 f1 := r.WriteFile("potato", "Potato Content", t1) 1196 f2 := r.WriteFile("sub/yam", "Yam Content", t2) 1197 1198 accounting.GlobalStats().ResetCounters() 1199 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1200 1201 fstest.CheckItems(t, r.Fremote, f1, f2) 1202 fstest.CheckItems(t, r.Flocal, f1, f2) 1203 1204 // Now rename locally. 1205 f2 = r.RenameFile(f2, "yam") 1206 1207 accounting.GlobalStats().ResetCounters() 1208 require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) 1209 1210 fstest.CheckItems(t, r.Fremote, f1, f2) 1211 1212 // Check we renamed something if we should have 1213 if canTrackRenames { 1214 renames := accounting.GlobalStats().Renames(0) 1215 assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) 1216 } 1217} 1218 1219func toyFileTransfers(r *fstest.Run) int64 { 1220 remote := r.Fremote.Name() 1221 transfers := 1 1222 if strings.HasPrefix(remote, "TestChunker") && strings.HasSuffix(remote, "S3") { 1223 transfers++ // Extra Copy because S3 emulates Move as Copy+Delete. 1224 } 1225 return int64(transfers) 1226} 1227 1228// Test a server-side move if possible, or the backup path if not 1229func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFilter, testDeleteEmptyDirs bool) { 1230 FremoteMove, _, finaliseMove, err := fstest.RandomRemote() 1231 require.NoError(t, err) 1232 defer finaliseMove() 1233 1234 file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) 1235 file2 := r.WriteBoth(ctx, "empty space", "-", t2) 1236 file3u := r.WriteBoth(ctx, "potato3", "------------------------------------------------------------ UPDATED", t2) 1237 1238 if testDeleteEmptyDirs { 1239 err := operations.Mkdir(ctx, r.Fremote, "tomatoDir") 1240 require.NoError(t, err) 1241 } 1242 1243 fstest.CheckItems(t, r.Fremote, file2, file1, file3u) 1244 1245 t.Logf("Server side move (if possible) %v -> %v", r.Fremote, FremoteMove) 1246 1247 // Write just one file in the new remote 1248 r.WriteObjectTo(ctx, FremoteMove, "empty space", "-", t2, false) 1249 file3 := r.WriteObjectTo(ctx, FremoteMove, "potato3", "------------------------------------------------------------", t1, false) 1250 fstest.CheckItems(t, FremoteMove, file2, file3) 1251 1252 // Do server-side move 1253 accounting.GlobalStats().ResetCounters() 1254 err = MoveDir(ctx, FremoteMove, r.Fremote, testDeleteEmptyDirs, false) 1255 require.NoError(t, err) 1256 1257 if withFilter { 1258 fstest.CheckItems(t, r.Fremote, file2) 1259 } else { 1260 fstest.CheckItems(t, r.Fremote) 1261 } 1262 1263 if testDeleteEmptyDirs { 1264 fstest.CheckListingWithPrecision(t, r.Fremote, nil, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 1265 } 1266 1267 fstest.CheckItems(t, FremoteMove, file2, file1, file3u) 1268 1269 // Create a new empty remote for stuff to be moved into 1270 FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote() 1271 require.NoError(t, err) 1272 defer finaliseMove2() 1273 1274 if testDeleteEmptyDirs { 1275 err := operations.Mkdir(ctx, FremoteMove, "tomatoDir") 1276 require.NoError(t, err) 1277 } 1278 1279 // Move it back to a new empty remote, dst does not exist this time 1280 accounting.GlobalStats().ResetCounters() 1281 err = MoveDir(ctx, FremoteMove2, FremoteMove, testDeleteEmptyDirs, false) 1282 require.NoError(t, err) 1283 1284 if withFilter { 1285 fstest.CheckItems(t, FremoteMove2, file1, file3u) 1286 fstest.CheckItems(t, FremoteMove, file2) 1287 } else { 1288 fstest.CheckItems(t, FremoteMove2, file2, file1, file3u) 1289 fstest.CheckItems(t, FremoteMove) 1290 } 1291 1292 if testDeleteEmptyDirs { 1293 fstest.CheckListingWithPrecision(t, FremoteMove, nil, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 1294 } 1295} 1296 1297// Test move 1298func TestMoveWithDeleteEmptySrcDirs(t *testing.T) { 1299 ctx := context.Background() 1300 r := fstest.NewRun(t) 1301 defer r.Finalise() 1302 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 1303 file2 := r.WriteFile("nested/sub dir/file", "nested", t1) 1304 r.Mkdir(ctx, r.Fremote) 1305 1306 // run move with --delete-empty-src-dirs 1307 err := MoveDir(ctx, r.Fremote, r.Flocal, true, false) 1308 require.NoError(t, err) 1309 1310 fstest.CheckListingWithPrecision( 1311 t, 1312 r.Flocal, 1313 nil, 1314 []string{}, 1315 fs.GetModifyWindow(ctx, r.Flocal), 1316 ) 1317 fstest.CheckItems(t, r.Fremote, file1, file2) 1318} 1319 1320func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) { 1321 ctx := context.Background() 1322 r := fstest.NewRun(t) 1323 defer r.Finalise() 1324 file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 1325 file2 := r.WriteFile("nested/sub dir/file", "nested", t1) 1326 r.Mkdir(ctx, r.Fremote) 1327 1328 err := MoveDir(ctx, r.Fremote, r.Flocal, false, false) 1329 require.NoError(t, err) 1330 1331 fstest.CheckListingWithPrecision( 1332 t, 1333 r.Flocal, 1334 nil, 1335 []string{ 1336 "sub dir", 1337 "nested", 1338 "nested/sub dir", 1339 }, 1340 fs.GetModifyWindow(ctx, r.Flocal), 1341 ) 1342 fstest.CheckItems(t, r.Fremote, file1, file2) 1343} 1344 1345func TestMoveWithIgnoreExisting(t *testing.T) { 1346 ctx := context.Background() 1347 ctx, ci := fs.AddConfig(ctx) 1348 r := fstest.NewRun(t) 1349 defer r.Finalise() 1350 file1 := r.WriteFile("existing", "potato", t1) 1351 file2 := r.WriteFile("existing-b", "tomato", t1) 1352 1353 ci.IgnoreExisting = true 1354 1355 accounting.GlobalStats().ResetCounters() 1356 err := MoveDir(ctx, r.Fremote, r.Flocal, false, false) 1357 require.NoError(t, err) 1358 fstest.CheckListingWithPrecision( 1359 t, 1360 r.Flocal, 1361 []fstest.Item{}, 1362 []string{}, 1363 fs.GetModifyWindow(ctx, r.Flocal), 1364 ) 1365 fstest.CheckListingWithPrecision( 1366 t, 1367 r.Fremote, 1368 []fstest.Item{ 1369 file1, 1370 file2, 1371 }, 1372 []string{}, 1373 fs.GetModifyWindow(ctx, r.Fremote), 1374 ) 1375 1376 // Recreate first file with modified content 1377 file1b := r.WriteFile("existing", "newpotatoes", t2) 1378 accounting.GlobalStats().ResetCounters() 1379 err = MoveDir(ctx, r.Fremote, r.Flocal, false, false) 1380 require.NoError(t, err) 1381 // Source items should still exist in modified state 1382 fstest.CheckListingWithPrecision( 1383 t, 1384 r.Flocal, 1385 []fstest.Item{ 1386 file1b, 1387 }, 1388 []string{}, 1389 fs.GetModifyWindow(ctx, r.Flocal), 1390 ) 1391 // Dest items should not have changed 1392 fstest.CheckListingWithPrecision( 1393 t, 1394 r.Fremote, 1395 []fstest.Item{ 1396 file1, 1397 file2, 1398 }, 1399 []string{}, 1400 fs.GetModifyWindow(ctx, r.Fremote), 1401 ) 1402} 1403 1404// Test a server-side move if possible, or the backup path if not 1405func TestServerSideMove(t *testing.T) { 1406 ctx := context.Background() 1407 r := fstest.NewRun(t) 1408 defer r.Finalise() 1409 testServerSideMove(ctx, t, r, false, false) 1410} 1411 1412// Test a server-side move if possible, or the backup path if not 1413func TestServerSideMoveWithFilter(t *testing.T) { 1414 ctx := context.Background() 1415 r := fstest.NewRun(t) 1416 defer r.Finalise() 1417 1418 fi, err := filter.NewFilter(nil) 1419 require.NoError(t, err) 1420 fi.Opt.MinSize = 40 1421 ctx = filter.ReplaceConfig(ctx, fi) 1422 1423 testServerSideMove(ctx, t, r, true, false) 1424} 1425 1426// Test a server-side move if possible 1427func TestServerSideMoveDeleteEmptySourceDirs(t *testing.T) { 1428 ctx := context.Background() 1429 r := fstest.NewRun(t) 1430 defer r.Finalise() 1431 testServerSideMove(ctx, t, r, false, true) 1432} 1433 1434// Test a server-side move with overlap 1435func TestServerSideMoveOverlap(t *testing.T) { 1436 ctx := context.Background() 1437 r := fstest.NewRun(t) 1438 defer r.Finalise() 1439 1440 if r.Fremote.Features().DirMove != nil { 1441 t.Skip("Skipping test as remote supports DirMove") 1442 } 1443 1444 subRemoteName := r.FremoteName + "/rclone-move-test" 1445 FremoteMove, err := fs.NewFs(ctx, subRemoteName) 1446 require.NoError(t, err) 1447 1448 file1 := r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1) 1449 fstest.CheckItems(t, r.Fremote, file1) 1450 1451 // Subdir move with no filters should return ErrorCantMoveOverlapping 1452 err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) 1453 assert.EqualError(t, err, fs.ErrorOverlapping.Error()) 1454 1455 // Now try with a filter which should also fail with ErrorCantMoveOverlapping 1456 fi, err := filter.NewFilter(nil) 1457 require.NoError(t, err) 1458 fi.Opt.MinSize = 40 1459 ctx = filter.ReplaceConfig(ctx, fi) 1460 1461 err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) 1462 assert.EqualError(t, err, fs.ErrorOverlapping.Error()) 1463} 1464 1465// Test a sync with overlap 1466func TestSyncOverlap(t *testing.T) { 1467 ctx := context.Background() 1468 r := fstest.NewRun(t) 1469 defer r.Finalise() 1470 1471 subRemoteName := r.FremoteName + "/rclone-sync-test" 1472 FremoteSync, err := fs.NewFs(ctx, subRemoteName) 1473 require.NoError(t, err) 1474 1475 checkErr := func(err error) { 1476 require.Error(t, err) 1477 assert.True(t, fserrors.IsFatalError(err)) 1478 assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error()) 1479 } 1480 1481 checkErr(Sync(ctx, FremoteSync, r.Fremote, false)) 1482 checkErr(Sync(ctx, r.Fremote, FremoteSync, false)) 1483 checkErr(Sync(ctx, r.Fremote, r.Fremote, false)) 1484 checkErr(Sync(ctx, FremoteSync, FremoteSync, false)) 1485} 1486 1487// Test with CompareDest set 1488func TestSyncCompareDest(t *testing.T) { 1489 ctx := context.Background() 1490 ctx, ci := fs.AddConfig(ctx) 1491 r := fstest.NewRun(t) 1492 defer r.Finalise() 1493 1494 ci.CompareDest = []string{r.FremoteName + "/CompareDest"} 1495 1496 fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") 1497 require.NoError(t, err) 1498 1499 // check empty dest, empty compare 1500 file1 := r.WriteFile("one", "one", t1) 1501 fstest.CheckItems(t, r.Flocal, file1) 1502 1503 accounting.GlobalStats().ResetCounters() 1504 err = Sync(ctx, fdst, r.Flocal, false) 1505 require.NoError(t, err) 1506 1507 file1dst := file1 1508 file1dst.Path = "dst/one" 1509 1510 fstest.CheckItems(t, r.Fremote, file1dst) 1511 1512 // check old dest, empty compare 1513 file1b := r.WriteFile("one", "onet2", t2) 1514 fstest.CheckItems(t, r.Fremote, file1dst) 1515 fstest.CheckItems(t, r.Flocal, file1b) 1516 1517 accounting.GlobalStats().ResetCounters() 1518 err = Sync(ctx, fdst, r.Flocal, false) 1519 require.NoError(t, err) 1520 1521 file1bdst := file1b 1522 file1bdst.Path = "dst/one" 1523 1524 fstest.CheckItems(t, r.Fremote, file1bdst) 1525 1526 // check old dest, new compare 1527 file3 := r.WriteObject(ctx, "dst/one", "one", t1) 1528 file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2) 1529 file1c := r.WriteFile("one", "onet2", t2) 1530 fstest.CheckItems(t, r.Fremote, file2, file3) 1531 fstest.CheckItems(t, r.Flocal, file1c) 1532 1533 accounting.GlobalStats().ResetCounters() 1534 err = Sync(ctx, fdst, r.Flocal, false) 1535 require.NoError(t, err) 1536 1537 fstest.CheckItems(t, r.Fremote, file2, file3) 1538 1539 // check empty dest, new compare 1540 file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2) 1541 file5 := r.WriteFile("two", "two", t2) 1542 fstest.CheckItems(t, r.Fremote, file2, file3, file4) 1543 fstest.CheckItems(t, r.Flocal, file1c, file5) 1544 1545 accounting.GlobalStats().ResetCounters() 1546 err = Sync(ctx, fdst, r.Flocal, false) 1547 require.NoError(t, err) 1548 1549 fstest.CheckItems(t, r.Fremote, file2, file3, file4) 1550 1551 // check new dest, new compare 1552 accounting.GlobalStats().ResetCounters() 1553 err = Sync(ctx, fdst, r.Flocal, false) 1554 require.NoError(t, err) 1555 1556 fstest.CheckItems(t, r.Fremote, file2, file3, file4) 1557 1558 // Work out if we actually have hashes for uploaded files 1559 haveHash := false 1560 if ht := fdst.Hashes().GetOne(); ht != hash.None { 1561 file2obj, err := fdst.NewObject(ctx, "one") 1562 if err == nil { 1563 file2objHash, err := file2obj.Hash(ctx, ht) 1564 if err == nil { 1565 haveHash = file2objHash != "" 1566 } 1567 } 1568 } 1569 1570 // check new dest, new compare, src timestamp differs 1571 // 1572 // we only check this if we the file we uploaded previously 1573 // actually has a hash otherwise the differing timestamp is 1574 // always copied. 1575 if haveHash { 1576 file5b := r.WriteFile("two", "two", t3) 1577 fstest.CheckItems(t, r.Flocal, file1c, file5b) 1578 1579 accounting.GlobalStats().ResetCounters() 1580 err = Sync(ctx, fdst, r.Flocal, false) 1581 require.NoError(t, err) 1582 1583 fstest.CheckItems(t, r.Fremote, file2, file3, file4) 1584 } else { 1585 t.Log("No hash on uploaded file so skipping compare timestamp test") 1586 } 1587 1588 // check empty dest, old compare 1589 file5c := r.WriteFile("two", "twot3", t3) 1590 fstest.CheckItems(t, r.Fremote, file2, file3, file4) 1591 fstest.CheckItems(t, r.Flocal, file1c, file5c) 1592 1593 accounting.GlobalStats().ResetCounters() 1594 err = Sync(ctx, fdst, r.Flocal, false) 1595 require.NoError(t, err) 1596 1597 file5cdst := file5c 1598 file5cdst.Path = "dst/two" 1599 1600 fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5cdst) 1601} 1602 1603// Test with multiple CompareDest 1604func TestSyncMultipleCompareDest(t *testing.T) { 1605 ctx := context.Background() 1606 ctx, ci := fs.AddConfig(ctx) 1607 r := fstest.NewRun(t) 1608 defer r.Finalise() 1609 1610 ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"} 1611 1612 // check empty dest, new compare 1613 fsrc1 := r.WriteFile("1", "1", t1) 1614 fsrc2 := r.WriteFile("2", "2", t1) 1615 fsrc3 := r.WriteFile("3", "3", t1) 1616 fstest.CheckItems(t, r.Flocal, fsrc1, fsrc2, fsrc3) 1617 1618 fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1) 1619 fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1) 1620 fstest.CheckItems(t, r.Fremote, fdest1, fdest2) 1621 1622 accounting.GlobalStats().ResetCounters() 1623 fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest") 1624 require.NoError(t, err) 1625 require.NoError(t, Sync(ctx, fdst, r.Flocal, false)) 1626 1627 fdest3 := fsrc3 1628 fdest3.Path = "dest/3" 1629 1630 fstest.CheckItems(t, fdst, fsrc3) 1631 fstest.CheckItems(t, r.Fremote, fdest1, fdest2, fdest3) 1632} 1633 1634// Test with CopyDest set 1635func TestSyncCopyDest(t *testing.T) { 1636 ctx := context.Background() 1637 ctx, ci := fs.AddConfig(ctx) 1638 r := fstest.NewRun(t) 1639 defer r.Finalise() 1640 1641 if r.Fremote.Features().Copy == nil { 1642 t.Skip("Skipping test as remote does not support server-side copy") 1643 } 1644 1645 ci.CopyDest = []string{r.FremoteName + "/CopyDest"} 1646 1647 fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") 1648 require.NoError(t, err) 1649 1650 // check empty dest, empty copy 1651 file1 := r.WriteFile("one", "one", t1) 1652 fstest.CheckItems(t, r.Flocal, file1) 1653 1654 accounting.GlobalStats().ResetCounters() 1655 err = Sync(ctx, fdst, r.Flocal, false) 1656 require.NoError(t, err) 1657 1658 file1dst := file1 1659 file1dst.Path = "dst/one" 1660 1661 fstest.CheckItems(t, r.Fremote, file1dst) 1662 1663 // check old dest, empty copy 1664 file1b := r.WriteFile("one", "onet2", t2) 1665 fstest.CheckItems(t, r.Fremote, file1dst) 1666 fstest.CheckItems(t, r.Flocal, file1b) 1667 1668 accounting.GlobalStats().ResetCounters() 1669 err = Sync(ctx, fdst, r.Flocal, false) 1670 require.NoError(t, err) 1671 1672 file1bdst := file1b 1673 file1bdst.Path = "dst/one" 1674 1675 fstest.CheckItems(t, r.Fremote, file1bdst) 1676 1677 // check old dest, new copy, backup-dir 1678 1679 ci.BackupDir = r.FremoteName + "/BackupDir" 1680 1681 file3 := r.WriteObject(ctx, "dst/one", "one", t1) 1682 file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2) 1683 file1c := r.WriteFile("one", "onet2", t2) 1684 fstest.CheckItems(t, r.Fremote, file2, file3) 1685 fstest.CheckItems(t, r.Flocal, file1c) 1686 1687 accounting.GlobalStats().ResetCounters() 1688 err = Sync(ctx, fdst, r.Flocal, false) 1689 require.NoError(t, err) 1690 1691 file2dst := file2 1692 file2dst.Path = "dst/one" 1693 file3.Path = "BackupDir/one" 1694 1695 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3) 1696 ci.BackupDir = "" 1697 1698 // check empty dest, new copy 1699 file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2) 1700 file5 := r.WriteFile("two", "two", t2) 1701 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4) 1702 fstest.CheckItems(t, r.Flocal, file1c, file5) 1703 1704 accounting.GlobalStats().ResetCounters() 1705 err = Sync(ctx, fdst, r.Flocal, false) 1706 require.NoError(t, err) 1707 1708 file4dst := file4 1709 file4dst.Path = "dst/two" 1710 1711 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) 1712 1713 // check new dest, new copy 1714 accounting.GlobalStats().ResetCounters() 1715 err = Sync(ctx, fdst, r.Flocal, false) 1716 require.NoError(t, err) 1717 1718 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) 1719 1720 // check empty dest, old copy 1721 file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2) 1722 file7 := r.WriteFile("three", "threet3", t3) 1723 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6) 1724 fstest.CheckItems(t, r.Flocal, file1c, file5, file7) 1725 1726 accounting.GlobalStats().ResetCounters() 1727 err = Sync(ctx, fdst, r.Flocal, false) 1728 require.NoError(t, err) 1729 1730 file7dst := file7 1731 file7dst.Path = "dst/three" 1732 1733 fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst) 1734} 1735 1736// Test with BackupDir set 1737func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) { 1738 ctx := context.Background() 1739 ctx, ci := fs.AddConfig(ctx) 1740 r := fstest.NewRun(t) 1741 defer r.Finalise() 1742 1743 if !operations.CanServerSideMove(r.Fremote) { 1744 t.Skip("Skipping test as remote does not support server-side move") 1745 } 1746 r.Mkdir(ctx, r.Fremote) 1747 1748 if backupDir != "" { 1749 ci.BackupDir = r.FremoteName + "/" + backupDir 1750 backupDir += "/" 1751 } else { 1752 ci.BackupDir = "" 1753 backupDir = "dst/" 1754 // Exclude the suffix from the sync otherwise the sync 1755 // deletes the old backup files 1756 flt, err := filter.NewFilter(nil) 1757 require.NoError(t, err) 1758 require.NoError(t, flt.AddRule("- *"+suffix)) 1759 // Change the active filter 1760 ctx = filter.ReplaceConfig(ctx, flt) 1761 } 1762 ci.Suffix = suffix 1763 ci.SuffixKeepExtension = suffixKeepExtension 1764 1765 // Make the setup so we have one, two, three in the dest 1766 // and one (different), two (same) in the source 1767 file1 := r.WriteObject(ctx, "dst/one", "one", t1) 1768 file2 := r.WriteObject(ctx, "dst/two", "two", t1) 1769 file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1) 1770 file2a := r.WriteFile("two", "two", t1) 1771 file1a := r.WriteFile("one", "oneA", t2) 1772 1773 fstest.CheckItems(t, r.Fremote, file1, file2, file3) 1774 fstest.CheckItems(t, r.Flocal, file1a, file2a) 1775 1776 fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") 1777 require.NoError(t, err) 1778 1779 accounting.GlobalStats().ResetCounters() 1780 err = Sync(ctx, fdst, r.Flocal, false) 1781 require.NoError(t, err) 1782 1783 // one should be moved to the backup dir and the new one installed 1784 file1.Path = backupDir + "one" + suffix 1785 file1a.Path = "dst/one" 1786 // two should be unchanged 1787 // three should be moved to the backup dir 1788 if suffixKeepExtension { 1789 file3.Path = backupDir + "three" + suffix + ".txt" 1790 } else { 1791 file3.Path = backupDir + "three.txt" + suffix 1792 } 1793 1794 fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a) 1795 1796 // Now check what happens if we do it again 1797 // Restore a different three and update one in the source 1798 file3a := r.WriteObject(ctx, "dst/three.txt", "threeA", t2) 1799 file1b := r.WriteFile("one", "oneBB", t3) 1800 fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a, file3a) 1801 1802 // This should delete three and overwrite one again, checking 1803 // the files got overwritten correctly in backup-dir 1804 accounting.GlobalStats().ResetCounters() 1805 err = Sync(ctx, fdst, r.Flocal, false) 1806 require.NoError(t, err) 1807 1808 // one should be moved to the backup dir and the new one installed 1809 file1a.Path = backupDir + "one" + suffix 1810 file1b.Path = "dst/one" 1811 // two should be unchanged 1812 // three should be moved to the backup dir 1813 if suffixKeepExtension { 1814 file3a.Path = backupDir + "three" + suffix + ".txt" 1815 } else { 1816 file3a.Path = backupDir + "three.txt" + suffix 1817 } 1818 1819 fstest.CheckItems(t, r.Fremote, file1b, file2, file3a, file1a) 1820} 1821func TestSyncBackupDir(t *testing.T) { 1822 testSyncBackupDir(t, "backup", "", false) 1823} 1824func TestSyncBackupDirWithSuffix(t *testing.T) { 1825 testSyncBackupDir(t, "backup", ".bak", false) 1826} 1827func TestSyncBackupDirWithSuffixKeepExtension(t *testing.T) { 1828 testSyncBackupDir(t, "backup", "-2019-01-01", true) 1829} 1830func TestSyncBackupDirSuffixOnly(t *testing.T) { 1831 testSyncBackupDir(t, "", ".bak", false) 1832} 1833 1834// Test with Suffix set 1835func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) { 1836 ctx := context.Background() 1837 ctx, ci := fs.AddConfig(ctx) 1838 r := fstest.NewRun(t) 1839 defer r.Finalise() 1840 1841 if !operations.CanServerSideMove(r.Fremote) { 1842 t.Skip("Skipping test as remote does not support server-side move") 1843 } 1844 r.Mkdir(ctx, r.Fremote) 1845 1846 ci.Suffix = suffix 1847 ci.SuffixKeepExtension = suffixKeepExtension 1848 1849 // Make the setup so we have one, two, three in the dest 1850 // and one (different), two (same) in the source 1851 file1 := r.WriteObject(ctx, "dst/one", "one", t1) 1852 file2 := r.WriteObject(ctx, "dst/two", "two", t1) 1853 file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1) 1854 file2a := r.WriteFile("two", "two", t1) 1855 file1a := r.WriteFile("one", "oneA", t2) 1856 file3a := r.WriteFile("three.txt", "threeA", t1) 1857 1858 fstest.CheckItems(t, r.Fremote, file1, file2, file3) 1859 fstest.CheckItems(t, r.Flocal, file1a, file2a, file3a) 1860 1861 fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") 1862 require.NoError(t, err) 1863 1864 accounting.GlobalStats().ResetCounters() 1865 err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one") 1866 require.NoError(t, err) 1867 err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two") 1868 require.NoError(t, err) 1869 err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt") 1870 require.NoError(t, err) 1871 1872 // one should be moved to the backup dir and the new one installed 1873 file1.Path = "dst/one" + suffix 1874 file1a.Path = "dst/one" 1875 // two should be unchanged 1876 // three should be moved to the backup dir 1877 if suffixKeepExtension { 1878 file3.Path = "dst/three" + suffix + ".txt" 1879 } else { 1880 file3.Path = "dst/three.txt" + suffix 1881 } 1882 file3a.Path = "dst/three.txt" 1883 1884 fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a, file3a) 1885 1886 // Now check what happens if we do it again 1887 // Restore a different three and update one in the source 1888 file3b := r.WriteFile("three.txt", "threeBDifferentSize", t3) 1889 file1b := r.WriteFile("one", "oneBB", t3) 1890 fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a, file3a) 1891 1892 // This should delete three and overwrite one again, checking 1893 // the files got overwritten correctly in backup-dir 1894 accounting.GlobalStats().ResetCounters() 1895 err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one") 1896 require.NoError(t, err) 1897 err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two") 1898 require.NoError(t, err) 1899 err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt") 1900 require.NoError(t, err) 1901 1902 // one should be moved to the backup dir and the new one installed 1903 file1a.Path = "dst/one" + suffix 1904 file1b.Path = "dst/one" 1905 // two should be unchanged 1906 // three should be moved to the backup dir 1907 if suffixKeepExtension { 1908 file3a.Path = "dst/three" + suffix + ".txt" 1909 } else { 1910 file3a.Path = "dst/three.txt" + suffix 1911 } 1912 file3b.Path = "dst/three.txt" 1913 1914 fstest.CheckItems(t, r.Fremote, file1b, file3b, file2, file3a, file1a) 1915} 1916func TestSyncSuffix(t *testing.T) { testSyncSuffix(t, ".bak", false) } 1917func TestSyncSuffixKeepExtension(t *testing.T) { testSyncSuffix(t, "-2019-01-01", true) } 1918 1919// Check we can sync two files with differing UTF-8 representations 1920func TestSyncUTFNorm(t *testing.T) { 1921 ctx := context.Background() 1922 if runtime.GOOS == "darwin" { 1923 t.Skip("Can't test UTF normalization on OS X") 1924 } 1925 1926 r := fstest.NewRun(t) 1927 defer r.Finalise() 1928 1929 // Two strings with different unicode normalization (from OS X) 1930 Encoding1 := "Testêé" 1931 Encoding2 := "Testêé" 1932 assert.NotEqual(t, Encoding1, Encoding2) 1933 assert.Equal(t, norm.NFC.String(Encoding1), norm.NFC.String(Encoding2)) 1934 1935 file1 := r.WriteFile(Encoding1, "This is a test", t1) 1936 fstest.CheckItems(t, r.Flocal, file1) 1937 1938 file2 := r.WriteObject(ctx, Encoding2, "This is a old test", t2) 1939 fstest.CheckItems(t, r.Fremote, file2) 1940 1941 accounting.GlobalStats().ResetCounters() 1942 err := Sync(ctx, r.Fremote, r.Flocal, false) 1943 require.NoError(t, err) 1944 1945 // We should have transferred exactly one file, but kept the 1946 // normalized state of the file. 1947 assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 1948 fstest.CheckItems(t, r.Flocal, file1) 1949 file1.Path = file2.Path 1950 fstest.CheckItems(t, r.Fremote, file1) 1951} 1952 1953// Test --immutable 1954func TestSyncImmutable(t *testing.T) { 1955 ctx := context.Background() 1956 ctx, ci := fs.AddConfig(ctx) 1957 r := fstest.NewRun(t) 1958 defer r.Finalise() 1959 1960 ci.Immutable = true 1961 1962 // Create file on source 1963 file1 := r.WriteFile("existing", "potato", t1) 1964 fstest.CheckItems(t, r.Flocal, file1) 1965 fstest.CheckItems(t, r.Fremote) 1966 1967 // Should succeed 1968 accounting.GlobalStats().ResetCounters() 1969 err := Sync(ctx, r.Fremote, r.Flocal, false) 1970 require.NoError(t, err) 1971 fstest.CheckItems(t, r.Flocal, file1) 1972 fstest.CheckItems(t, r.Fremote, file1) 1973 1974 // Modify file data and timestamp on source 1975 file2 := r.WriteFile("existing", "tomatoes", t2) 1976 fstest.CheckItems(t, r.Flocal, file2) 1977 fstest.CheckItems(t, r.Fremote, file1) 1978 1979 // Should fail with ErrorImmutableModified and not modify local or remote files 1980 accounting.GlobalStats().ResetCounters() 1981 err = Sync(ctx, r.Fremote, r.Flocal, false) 1982 assert.EqualError(t, err, fs.ErrorImmutableModified.Error()) 1983 fstest.CheckItems(t, r.Flocal, file2) 1984 fstest.CheckItems(t, r.Fremote, file1) 1985} 1986 1987// Test --ignore-case-sync 1988func TestSyncIgnoreCase(t *testing.T) { 1989 ctx := context.Background() 1990 ctx, ci := fs.AddConfig(ctx) 1991 r := fstest.NewRun(t) 1992 defer r.Finalise() 1993 1994 // Only test if filesystems are case sensitive 1995 if r.Fremote.Features().CaseInsensitive || r.Flocal.Features().CaseInsensitive { 1996 t.Skip("Skipping test as local or remote are case-insensitive") 1997 } 1998 1999 ci.IgnoreCaseSync = true 2000 2001 // Create files with different filename casing 2002 file1 := r.WriteFile("existing", "potato", t1) 2003 fstest.CheckItems(t, r.Flocal, file1) 2004 file2 := r.WriteObject(ctx, "EXISTING", "potato", t1) 2005 fstest.CheckItems(t, r.Fremote, file2) 2006 2007 // Should not copy files that are differently-cased but otherwise identical 2008 accounting.GlobalStats().ResetCounters() 2009 err := Sync(ctx, r.Fremote, r.Flocal, false) 2010 require.NoError(t, err) 2011 fstest.CheckItems(t, r.Flocal, file1) 2012 fstest.CheckItems(t, r.Fremote, file2) 2013} 2014 2015// Test that aborting on --max-transfer works 2016func TestMaxTransfer(t *testing.T) { 2017 ctx := context.Background() 2018 ctx, ci := fs.AddConfig(ctx) 2019 ci.MaxTransfer = 3 * 1024 2020 ci.Transfers = 1 2021 ci.Checkers = 1 2022 ci.CutoffMode = fs.CutoffModeHard 2023 2024 test := func(t *testing.T, cutoff fs.CutoffMode) { 2025 r := fstest.NewRun(t) 2026 defer r.Finalise() 2027 ci.CutoffMode = cutoff 2028 2029 if r.Fremote.Name() != "local" { 2030 t.Skip("This test only runs on local") 2031 } 2032 2033 // Create file on source 2034 file1 := r.WriteFile("file1", string(make([]byte, 5*1024)), t1) 2035 file2 := r.WriteFile("file2", string(make([]byte, 2*1024)), t1) 2036 file3 := r.WriteFile("file3", string(make([]byte, 3*1024)), t1) 2037 fstest.CheckItems(t, r.Flocal, file1, file2, file3) 2038 fstest.CheckItems(t, r.Fremote) 2039 2040 accounting.GlobalStats().ResetCounters() 2041 2042 err := Sync(ctx, r.Fremote, r.Flocal, false) 2043 expectedErr := fserrors.FsError(accounting.ErrorMaxTransferLimitReachedFatal) 2044 if cutoff != fs.CutoffModeHard { 2045 expectedErr = accounting.ErrorMaxTransferLimitReachedGraceful 2046 } 2047 fserrors.Count(expectedErr) 2048 assert.Equal(t, expectedErr, err) 2049 } 2050 2051 t.Run("Hard", func(t *testing.T) { test(t, fs.CutoffModeHard) }) 2052 t.Run("Soft", func(t *testing.T) { test(t, fs.CutoffModeSoft) }) 2053 t.Run("Cautious", func(t *testing.T) { test(t, fs.CutoffModeCautious) }) 2054} 2055 2056func testSyncConcurrent(t *testing.T, subtest string) { 2057 const ( 2058 NFILES = 20 2059 NCHECKERS = 4 2060 NTRANSFERS = 4 2061 ) 2062 2063 ctx, ci := fs.AddConfig(context.Background()) 2064 ci.Checkers = NCHECKERS 2065 ci.Transfers = NTRANSFERS 2066 2067 r := fstest.NewRun(t) 2068 defer r.Finalise() 2069 stats := accounting.GlobalStats() 2070 2071 itemsBefore := []fstest.Item{} 2072 itemsAfter := []fstest.Item{} 2073 for i := 0; i < NFILES; i++ { 2074 nameBoth := fmt.Sprintf("both%d", i) 2075 nameOnly := fmt.Sprintf("only%d", i) 2076 switch subtest { 2077 case "delete": 2078 fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1) 2079 fileOnly := r.WriteObject(ctx, nameOnly, "potato", t1) 2080 itemsBefore = append(itemsBefore, fileBoth, fileOnly) 2081 itemsAfter = append(itemsAfter, fileBoth) 2082 case "truncate": 2083 fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1) 2084 fileFull := r.WriteObject(ctx, nameOnly, "potato", t1) 2085 fileEmpty := r.WriteFile(nameOnly, "", t1) 2086 itemsBefore = append(itemsBefore, fileBoth, fileFull) 2087 itemsAfter = append(itemsAfter, fileBoth, fileEmpty) 2088 } 2089 } 2090 2091 fstest.CheckItems(t, r.Fremote, itemsBefore...) 2092 stats.ResetErrors() 2093 err := Sync(ctx, r.Fremote, r.Flocal, false) 2094 if err == fs.ErrorCantUploadEmptyFiles { 2095 t.Skipf("Skip test because remote cannot upload empty files") 2096 } 2097 assert.NoError(t, err, "Sync must not return a error") 2098 assert.False(t, stats.Errored(), "Low level errors must not have happened") 2099 fstest.CheckItems(t, r.Fremote, itemsAfter...) 2100} 2101 2102func TestSyncConcurrentDelete(t *testing.T) { 2103 testSyncConcurrent(t, "delete") 2104} 2105 2106func TestSyncConcurrentTruncate(t *testing.T) { 2107 testSyncConcurrent(t, "truncate") 2108} 2109