1// Copyright (C) 2014 The Syncthing Authors. 2// 3// This Source Code Form is subject to the terms of the Mozilla Public 4// License, v. 2.0. If a copy of the MPL was not distributed with this file, 5// You can obtain one at https://mozilla.org/MPL/2.0/. 6 7package db 8 9import ( 10 "bytes" 11 "context" 12 "fmt" 13 "testing" 14 15 "github.com/syncthing/syncthing/lib/db/backend" 16 "github.com/syncthing/syncthing/lib/events" 17 "github.com/syncthing/syncthing/lib/fs" 18 "github.com/syncthing/syncthing/lib/protocol" 19) 20 21func genBlocks(n int) []protocol.BlockInfo { 22 b := make([]protocol.BlockInfo, n) 23 for i := range b { 24 h := make([]byte, 32) 25 for j := range h { 26 h[j] = byte(i + j) 27 } 28 b[i].Size = i 29 b[i].Hash = h 30 } 31 return b 32} 33 34func TestIgnoredFiles(t *testing.T) { 35 ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons") 36 if err != nil { 37 t.Fatal(err) 38 } 39 db := newLowlevel(t, ldb) 40 defer db.Close() 41 if err := UpdateSchema(db); err != nil { 42 t.Fatal(err) 43 } 44 45 fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) 46 47 // The contents of the database are like this: 48 // 49 // fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) 50 // fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ 51 // { // invalid (ignored) file 52 // Name: "foo", 53 // Type: protocol.FileInfoTypeFile, 54 // Invalid: true, 55 // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, 56 // }, 57 // { // regular file 58 // Name: "bar", 59 // Type: protocol.FileInfoTypeFile, 60 // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, 61 // }, 62 // }) 63 // fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ 64 // { // invalid file 65 // Name: "baz", 66 // Type: protocol.FileInfoTypeFile, 67 // Invalid: true, 68 // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, 69 // }, 70 // { // regular file 71 // Name: "quux", 72 // Type: protocol.FileInfoTypeFile, 73 // Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, 74 // }, 75 // }) 76 77 // Local files should have the "ignored" bit in addition to just being 78 // generally invalid if we want to look at the simulation of that bit. 79 80 snap := snapshot(t, fs) 81 defer snap.Release() 82 fi, ok := snap.Get(protocol.LocalDeviceID, "foo") 83 if !ok { 84 t.Fatal("foo should exist") 85 } 86 if !fi.IsInvalid() { 87 t.Error("foo should be invalid") 88 } 89 if !fi.IsIgnored() { 90 t.Error("foo should be ignored") 91 } 92 93 fi, ok = snap.Get(protocol.LocalDeviceID, "bar") 94 if !ok { 95 t.Fatal("bar should exist") 96 } 97 if fi.IsInvalid() { 98 t.Error("bar should not be invalid") 99 } 100 if fi.IsIgnored() { 101 t.Error("bar should not be ignored") 102 } 103 104 // Remote files have the invalid bit as usual, and the IsInvalid() method 105 // should pick this up too. 106 107 fi, ok = snap.Get(protocol.DeviceID{42}, "baz") 108 if !ok { 109 t.Fatal("baz should exist") 110 } 111 if !fi.IsInvalid() { 112 t.Error("baz should be invalid") 113 } 114 if !fi.IsInvalid() { 115 t.Error("baz should be invalid") 116 } 117 118 fi, ok = snap.Get(protocol.DeviceID{42}, "quux") 119 if !ok { 120 t.Fatal("quux should exist") 121 } 122 if fi.IsInvalid() { 123 t.Error("quux should not be invalid") 124 } 125 if fi.IsInvalid() { 126 t.Error("quux should not be invalid") 127 } 128} 129 130const myID = 1 131 132var ( 133 remoteDevice0, remoteDevice1 protocol.DeviceID 134 update0to3Folder = "UpdateSchema0to3" 135 invalid = "invalid" 136 slashPrefixed = "/notgood" 137 haveUpdate0to3 map[protocol.DeviceID]fileList 138) 139 140func init() { 141 remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") 142 remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU") 143 haveUpdate0to3 = map[protocol.DeviceID]fileList{ 144 protocol.LocalDeviceID: { 145 protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, 146 protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, 147 }, 148 remoteDevice0: { 149 protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, 150 protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true}, 151 protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)}, 152 }, 153 remoteDevice1: { 154 protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)}, 155 protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), RawInvalid: true}, 156 protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true}, 157 }, 158 } 159} 160 161func TestUpdate0to3(t *testing.T) { 162 ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons") 163 164 if err != nil { 165 t.Fatal(err) 166 } 167 168 db := newLowlevel(t, ldb) 169 defer db.Close() 170 updater := schemaUpdater{db} 171 172 folder := []byte(update0to3Folder) 173 174 if err := updater.updateSchema0to1(0); err != nil { 175 t.Fatal(err) 176 } 177 178 trans, err := db.newReadOnlyTransaction() 179 if err != nil { 180 t.Fatal(err) 181 } 182 defer trans.Release() 183 if _, ok, err := trans.getFile(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil { 184 t.Fatal(err) 185 } else if ok { 186 t.Error("File prefixed by '/' was not removed during transition to schema 1") 187 } 188 189 var key []byte 190 191 key, err = db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)) 192 if err != nil { 193 t.Fatal(err) 194 } 195 if _, err := db.Get(key); err != nil { 196 t.Error("Invalid file wasn't added to global list") 197 } 198 199 if err := updater.updateSchema1to2(1); err != nil { 200 t.Fatal(err) 201 } 202 203 found := false 204 trans, err = db.newReadOnlyTransaction() 205 if err != nil { 206 t.Fatal(err) 207 } 208 defer trans.Release() 209 _ = trans.withHaveSequence(folder, 0, func(fi protocol.FileIntf) bool { 210 f := fi.(protocol.FileInfo) 211 l.Infoln(f) 212 if found { 213 t.Error("Unexpected additional file via sequence", f.FileName()) 214 return true 215 } 216 if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, 0, true, true, 0) { 217 found = true 218 } else { 219 t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) 220 } 221 return true 222 }) 223 if !found { 224 t.Error("Local file wasn't added to sequence bucket", err) 225 } 226 227 if err := updater.updateSchema2to3(2); err != nil { 228 t.Fatal(err) 229 } 230 231 need := map[string]protocol.FileInfo{ 232 haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0], 233 haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0], 234 haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2], 235 } 236 237 trans, err = db.newReadOnlyTransaction() 238 if err != nil { 239 t.Fatal(err) 240 } 241 defer trans.Release() 242 243 key, err = trans.keyer.GenerateNeedFileKey(nil, folder, nil) 244 if err != nil { 245 t.Fatal(err) 246 } 247 dbi, err := trans.NewPrefixIterator(key) 248 if err != nil { 249 t.Fatal(err) 250 } 251 defer dbi.Release() 252 253 for dbi.Next() { 254 name := trans.keyer.NameFromGlobalVersionKey(dbi.Key()) 255 key, err = trans.keyer.GenerateGlobalVersionKey(key, folder, name) 256 bs, err := trans.Get(key) 257 if err != nil { 258 t.Fatal(err) 259 } 260 var vl VersionListDeprecated 261 if err := vl.Unmarshal(bs); err != nil { 262 t.Fatal(err) 263 } 264 key, err = trans.keyer.GenerateDeviceFileKey(key, folder, vl.Versions[0].Device, name) 265 if err != nil { 266 t.Fatal(err) 267 } 268 fi, ok, err := trans.getFileTrunc(key, false) 269 if err != nil { 270 t.Fatal(err) 271 } 272 if !ok { 273 device := "<invalid>" 274 if dev, err := protocol.DeviceIDFromBytes(vl.Versions[0].Device); err != nil { 275 device = dev.String() 276 } 277 t.Fatal("surprise missing global file", string(name), device) 278 } 279 e, ok := need[fi.FileName()] 280 if !ok { 281 t.Error("Got unexpected needed file:", fi.FileName()) 282 } 283 f := fi.(protocol.FileInfo) 284 delete(need, f.Name) 285 if !f.IsEquivalentOptional(e, 0, true, true, 0) { 286 t.Errorf("Wrong needed file, got %v, expected %v", f, e) 287 } 288 } 289 if dbi.Error() != nil { 290 t.Fatal(err) 291 } 292 293 for n := range need { 294 t.Errorf(`Missing needed file "%v"`, n) 295 } 296} 297 298// TestRepairSequence checks that a few hand-crafted messed-up sequence entries get fixed. 299func TestRepairSequence(t *testing.T) { 300 db := newLowlevelMemory(t) 301 defer db.Close() 302 303 folderStr := "test" 304 folder := []byte(folderStr) 305 id := protocol.LocalDeviceID 306 short := protocol.LocalDeviceID.Short() 307 308 files := []protocol.FileInfo{ 309 {Name: "fine", Blocks: genBlocks(1)}, 310 {Name: "duplicate", Blocks: genBlocks(2)}, 311 {Name: "missing", Blocks: genBlocks(3)}, 312 {Name: "overwriting", Blocks: genBlocks(4)}, 313 {Name: "inconsistent", Blocks: genBlocks(5)}, 314 {Name: "inconsistentNotIndirected", Blocks: genBlocks(2)}, 315 } 316 for i, f := range files { 317 files[i].Version = f.Version.Update(short) 318 } 319 320 trans, err := db.newReadWriteTransaction() 321 if err != nil { 322 t.Fatal(err) 323 } 324 defer trans.close() 325 326 addFile := func(f protocol.FileInfo, seq int64) { 327 dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name)) 328 if err != nil { 329 t.Fatal(err) 330 } 331 if err := trans.putFile(dk, f); err != nil { 332 t.Fatal(err) 333 } 334 sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq) 335 if err != nil { 336 t.Fatal(err) 337 } 338 if err := trans.Put(sk, dk); err != nil { 339 t.Fatal(err) 340 } 341 } 342 343 // Plain normal entry 344 var seq int64 = 1 345 files[0].Sequence = 1 346 addFile(files[0], seq) 347 348 // Second entry once updated with original sequence still in place 349 f := files[1] 350 f.Sequence = int64(len(files) + 1) 351 addFile(f, f.Sequence) 352 // Original sequence entry 353 seq++ 354 sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq) 355 if err != nil { 356 t.Fatal(err) 357 } 358 dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name)) 359 if err != nil { 360 t.Fatal(err) 361 } 362 if err := trans.Put(sk, dk); err != nil { 363 t.Fatal(err) 364 } 365 366 // File later overwritten thus missing sequence entry 367 seq++ 368 files[2].Sequence = seq 369 addFile(files[2], seq) 370 371 // File overwriting previous sequence entry (no seq bump) 372 seq++ 373 files[3].Sequence = seq 374 addFile(files[3], seq) 375 376 // Inconistent files 377 seq++ 378 files[4].Sequence = 101 379 addFile(files[4], seq) 380 seq++ 381 files[5].Sequence = 102 382 addFile(files[5], seq) 383 384 // And a sequence entry pointing at nothing because why not 385 sk, err = trans.keyer.GenerateSequenceKey(nil, folder, 100001) 386 if err != nil { 387 t.Fatal(err) 388 } 389 dk, err = trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte("nonexisting")) 390 if err != nil { 391 t.Fatal(err) 392 } 393 if err := trans.Put(sk, dk); err != nil { 394 t.Fatal(err) 395 } 396 397 if err := trans.Commit(); err != nil { 398 t.Fatal(err) 399 } 400 401 // Loading the metadata for the first time means a "re"calculation happens, 402 // along which the sequences get repaired too. 403 db.gcMut.RLock() 404 _, err = db.loadMetadataTracker(folderStr) 405 db.gcMut.RUnlock() 406 if err != nil { 407 t.Fatal(err) 408 } 409 410 // Check the db 411 ro, err := db.newReadOnlyTransaction() 412 if err != nil { 413 t.Fatal(err) 414 } 415 defer ro.close() 416 417 it, err := ro.NewPrefixIterator([]byte{KeyTypeDevice}) 418 if err != nil { 419 t.Fatal(err) 420 } 421 defer it.Release() 422 for it.Next() { 423 fi, err := ro.unmarshalTrunc(it.Value(), true) 424 if err != nil { 425 t.Fatal(err) 426 } 427 if sk, err = ro.keyer.GenerateSequenceKey(sk, folder, fi.SequenceNo()); err != nil { 428 t.Fatal(err) 429 } 430 dk, err := ro.Get(sk) 431 if backend.IsNotFound(err) { 432 t.Error("Missing sequence entry for", fi.FileName()) 433 } else if err != nil { 434 t.Fatal(err) 435 } 436 if !bytes.Equal(it.Key(), dk) { 437 t.Errorf("Wrong key for %v, expected %s, got %s", f.FileName(), it.Key(), dk) 438 } 439 } 440 if err := it.Error(); err != nil { 441 t.Fatal(err) 442 } 443 it.Release() 444 445 it, err = ro.NewPrefixIterator([]byte{KeyTypeSequence}) 446 if err != nil { 447 t.Fatal(err) 448 } 449 defer it.Release() 450 for it.Next() { 451 intf, ok, err := ro.getFileTrunc(it.Value(), false) 452 if err != nil { 453 t.Fatal(err) 454 } 455 fi := intf.(protocol.FileInfo) 456 seq := ro.keyer.SequenceFromSequenceKey(it.Key()) 457 if !ok { 458 t.Errorf("Sequence entry %v points at nothing", seq) 459 } else if fi.SequenceNo() != seq { 460 t.Errorf("Inconsistent sequence entry for %v: %v != %v", fi.FileName(), fi.SequenceNo(), seq) 461 } 462 if len(fi.Blocks) == 0 { 463 t.Error("Missing blocks in", fi.FileName()) 464 } 465 } 466 if err := it.Error(); err != nil { 467 t.Fatal(err) 468 } 469 it.Release() 470} 471 472func TestDowngrade(t *testing.T) { 473 db := newLowlevelMemory(t) 474 defer db.Close() 475 // sets the min version etc 476 if err := UpdateSchema(db); err != nil { 477 t.Fatal(err) 478 } 479 480 // Bump the database version to something newer than we actually support 481 miscDB := NewMiscDataNamespace(db) 482 if err := miscDB.PutInt64("dbVersion", dbVersion+1); err != nil { 483 t.Fatal(err) 484 } 485 l.Infoln(dbVersion) 486 487 // Pretend we just opened the DB and attempt to update it again 488 err := UpdateSchema(db) 489 490 if err, ok := err.(*databaseDowngradeError); !ok { 491 t.Fatal("Expected error due to database downgrade, got", err) 492 } else if err.minSyncthingVersion != dbMinSyncthingVersion { 493 t.Fatalf("Error has %v as min Syncthing version, expected %v", err.minSyncthingVersion, dbMinSyncthingVersion) 494 } 495} 496 497func TestCheckGlobals(t *testing.T) { 498 db := newLowlevelMemory(t) 499 defer db.Close() 500 501 fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) 502 503 // Add any file 504 name := "foo" 505 fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ 506 { 507 Name: name, 508 Type: protocol.FileInfoTypeFile, 509 Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, 510 }, 511 }) 512 513 // Remove just the file entry 514 if err := db.dropPrefix([]byte{KeyTypeDevice}); err != nil { 515 t.Fatal(err) 516 } 517 518 // Clean up global entry of the now missing file 519 if repaired, err := db.checkGlobals(fs.folder); err != nil { 520 t.Fatal(err) 521 } else if repaired != 1 { 522 t.Error("Expected 1 repaired global item, got", repaired) 523 } 524 525 // Check that the global entry is gone 526 gk, err := db.keyer.GenerateGlobalVersionKey(nil, []byte(fs.folder), []byte(name)) 527 if err != nil { 528 t.Fatal(err) 529 } 530 _, err = db.Get(gk) 531 if !backend.IsNotFound(err) { 532 t.Error("Expected key missing error, got", err) 533 } 534} 535 536func TestUpdateTo10(t *testing.T) { 537 ldb, err := openJSONS("./testdata/v1.4.0-updateTo10.json") 538 if err != nil { 539 t.Fatal(err) 540 } 541 db := newLowlevel(t, ldb) 542 defer db.Close() 543 544 UpdateSchema(db) 545 546 folder := "test" 547 548 meta, err := db.getMetaAndCheck(folder) 549 if err != nil { 550 t.Fatal(err) 551 } 552 553 empty := Counts{} 554 555 c := meta.Counts(protocol.LocalDeviceID, needFlag) 556 if c.Files != 1 { 557 t.Error("Expected 1 needed file locally, got", c.Files) 558 } 559 c.Files = 0 560 if c.Deleted != 1 { 561 t.Error("Expected 1 needed deletion locally, got", c.Deleted) 562 } 563 c.Deleted = 0 564 if !c.Equal(empty) { 565 t.Error("Expected all counts to be zero, got", c) 566 } 567 c = meta.Counts(remoteDevice0, needFlag) 568 if !c.Equal(empty) { 569 t.Error("Expected all counts to be zero, got", c) 570 } 571 572 trans, err := db.newReadOnlyTransaction() 573 if err != nil { 574 t.Fatal(err) 575 } 576 defer trans.Release() 577 // a 578 vl, err := trans.getGlobalVersions(nil, []byte(folder), []byte("a")) 579 if err != nil { 580 t.Fatal(err) 581 } 582 for _, v := range vl.RawVersions { 583 if !v.Deleted { 584 t.Error("Unexpected undeleted global version for a") 585 } 586 } 587 // b 588 vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("b")) 589 if err != nil { 590 t.Fatal(err) 591 } 592 if !vl.RawVersions[0].Deleted { 593 t.Error("vl.Versions[0] not deleted for b") 594 } 595 if vl.RawVersions[1].Deleted { 596 t.Error("vl.Versions[1] deleted for b") 597 } 598 // c 599 vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("c")) 600 if err != nil { 601 t.Fatal(err) 602 } 603 if vl.RawVersions[0].Deleted { 604 t.Error("vl.Versions[0] deleted for c") 605 } 606 if !vl.RawVersions[1].Deleted { 607 t.Error("vl.Versions[1] not deleted for c") 608 } 609} 610 611func TestDropDuplicates(t *testing.T) { 612 names := []string{ 613 "foo", 614 "bar", 615 "dcxvoijnds", 616 "3d/dsfase/4/ss2", 617 } 618 tcs := []struct{ in, out []int }{ 619 {[]int{0}, []int{0}}, 620 {[]int{0, 1}, []int{0, 1}}, 621 {[]int{0, 1, 0, 1}, []int{0, 1}}, 622 {[]int{0, 1, 1, 1, 1}, []int{0, 1}}, 623 {[]int{0, 0, 0, 1}, []int{0, 1}}, 624 {[]int{0, 1, 2, 3}, []int{0, 1, 2, 3}}, 625 {[]int{3, 2, 1, 0, 0, 1, 2, 3}, []int{0, 1, 2, 3}}, 626 {[]int{0, 1, 1, 3, 0, 1, 0, 1, 2, 3}, []int{0, 1, 2, 3}}, 627 } 628 629 for tci, tc := range tcs { 630 inp := make([]protocol.FileInfo, len(tc.in)) 631 expSeq := make(map[string]int) 632 for i, j := range tc.in { 633 inp[i] = protocol.FileInfo{Name: names[j], Sequence: int64(i)} 634 expSeq[names[j]] = i 635 } 636 outp := normalizeFilenamesAndDropDuplicates(inp) 637 if len(outp) != len(tc.out) { 638 t.Errorf("tc %v: Expected %v entries, got %v", tci, len(tc.out), len(outp)) 639 continue 640 } 641 for i, f := range outp { 642 if exp := names[tc.out[i]]; exp != f.Name { 643 t.Errorf("tc %v: Got file %v at pos %v, expected %v", tci, f.Name, i, exp) 644 } 645 if exp := int64(expSeq[outp[i].Name]); exp != f.Sequence { 646 t.Errorf("tc %v: Got sequence %v at pos %v, expected %v", tci, f.Sequence, i, exp) 647 } 648 } 649 } 650} 651 652func TestGCIndirect(t *testing.T) { 653 // Verify that the gcIndirect run actually removes block lists. 654 655 db := newLowlevelMemory(t) 656 defer db.Close() 657 meta := newMetadataTracker(db.keyer, events.NoopLogger) 658 659 // Add three files with different block lists 660 661 files := []protocol.FileInfo{ 662 {Name: "a", Blocks: genBlocks(100)}, 663 {Name: "b", Blocks: genBlocks(200)}, 664 {Name: "c", Blocks: genBlocks(300)}, 665 } 666 667 db.updateLocalFiles([]byte("folder"), files, meta) 668 669 // Run a GC pass 670 671 db.gcIndirect(context.Background()) 672 673 // Verify that we have three different block lists 674 675 n, err := numBlockLists(db) 676 if err != nil { 677 t.Fatal(err) 678 } 679 if n != len(files) { 680 t.Fatal("expected each file to have a block list") 681 } 682 683 // Change the block lists for each file 684 685 for i := range files { 686 files[i].Version = files[i].Version.Update(42) 687 files[i].Blocks = genBlocks(len(files[i].Blocks) + 1) 688 } 689 690 db.updateLocalFiles([]byte("folder"), files, meta) 691 692 // Verify that we now have *six* different block lists 693 694 n, err = numBlockLists(db) 695 if err != nil { 696 t.Fatal(err) 697 } 698 if n != 2*len(files) { 699 t.Fatal("expected both old and new block lists to exist") 700 } 701 702 // Run a GC pass 703 704 db.gcIndirect(context.Background()) 705 706 // Verify that we now have just the three we need, again 707 708 n, err = numBlockLists(db) 709 if err != nil { 710 t.Fatal(err) 711 } 712 if n != len(files) { 713 t.Fatal("expected GC to collect all but the needed ones") 714 } 715 716 // Double check the correctness by loading the block lists and comparing with what we stored 717 718 tr, err := db.newReadOnlyTransaction() 719 if err != nil { 720 t.Fatal() 721 } 722 defer tr.Release() 723 for _, f := range files { 724 fi, ok, err := tr.getFile([]byte("folder"), protocol.LocalDeviceID[:], []byte(f.Name)) 725 if err != nil { 726 t.Fatal(err) 727 } 728 if !ok { 729 t.Fatal("mysteriously missing") 730 } 731 if len(fi.Blocks) != len(f.Blocks) { 732 t.Fatal("block list mismatch") 733 } 734 for i := range fi.Blocks { 735 if !bytes.Equal(fi.Blocks[i].Hash, f.Blocks[i].Hash) { 736 t.Fatal("hash mismatch") 737 } 738 } 739 } 740} 741 742func TestUpdateTo14(t *testing.T) { 743 db := newLowlevelMemory(t) 744 defer db.Close() 745 746 folderStr := "default" 747 folder := []byte(folderStr) 748 name := []byte("foo") 749 file := protocol.FileInfo{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(blocksIndirectionCutoff - 1)} 750 file.BlocksHash = protocol.BlocksHash(file.Blocks) 751 fileWOBlocks := file 752 fileWOBlocks.Blocks = nil 753 meta, err := db.loadMetadataTracker(folderStr) 754 if err != nil { 755 t.Fatal(err) 756 } 757 758 // Initally add the correct file the usual way, all good here. 759 if err := db.updateLocalFiles(folder, []protocol.FileInfo{file}, meta); err != nil { 760 t.Fatal(err) 761 } 762 763 // Simulate the previous bug, where .putFile could write a file info without 764 // blocks, even though the file has them (and thus a non-nil BlocksHash). 765 trans, err := db.newReadWriteTransaction() 766 if err != nil { 767 t.Fatal(err) 768 } 769 defer trans.close() 770 key, err := db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], name) 771 if err != nil { 772 t.Fatal(err) 773 } 774 fiBs := mustMarshal(&fileWOBlocks) 775 if err := trans.Put(key, fiBs); err != nil { 776 t.Fatal(err) 777 } 778 if err := trans.Commit(); err != nil { 779 t.Fatal(err) 780 } 781 trans.close() 782 783 // Run migration, pretending were still on schema 13. 784 if err := (&schemaUpdater{db}).updateSchemaTo14(13); err != nil { 785 t.Fatal(err) 786 } 787 788 // checks 789 ro, err := db.newReadOnlyTransaction() 790 if err != nil { 791 t.Fatal(err) 792 } 793 defer ro.close() 794 if f, ok, err := ro.getFileByKey(key); err != nil { 795 t.Fatal(err) 796 } else if !ok { 797 t.Error("file missing") 798 } else if !f.MustRescan() { 799 t.Error("file not marked as MustRescan") 800 } 801 802 if vl, err := ro.getGlobalVersions(nil, folder, name); err != nil { 803 t.Fatal(err) 804 } else if fv, ok := vl.GetGlobal(); !ok { 805 t.Error("missing global") 806 } else if !fv.IsInvalid() { 807 t.Error("global not marked as invalid") 808 } 809} 810 811func TestFlushRecursion(t *testing.T) { 812 // Verify that a commit hook can write to the transaction without 813 // causing another flush and thus recursion. 814 815 db := newLowlevelMemory(t) 816 defer db.Close() 817 818 // A commit hook that writes a small piece of data to the transaction. 819 hookFired := 0 820 hook := func(tx backend.WriteTransaction) error { 821 err := tx.Put([]byte(fmt.Sprintf("hook-key-%d", hookFired)), []byte(fmt.Sprintf("hook-value-%d", hookFired))) 822 if err != nil { 823 t.Fatal(err) 824 } 825 hookFired++ 826 return nil 827 } 828 829 // A transaction. 830 tx, err := db.NewWriteTransaction(hook) 831 if err != nil { 832 t.Fatal(err) 833 } 834 defer tx.Release() 835 836 // Write stuff until the transaction flushes, thus firing the hook. 837 i := 0 838 for hookFired == 0 { 839 err := tx.Put([]byte(fmt.Sprintf("key-%d", i)), []byte(fmt.Sprintf("value-%d", i))) 840 if err != nil { 841 t.Fatal(err) 842 } 843 i++ 844 } 845 846 // The hook should have fired precisely once. 847 if hookFired != 1 { 848 t.Error("expect one hook fire, not", hookFired) 849 } 850} 851 852func TestCheckLocalNeed(t *testing.T) { 853 db := newLowlevelMemory(t) 854 defer db.Close() 855 856 folderStr := "test" 857 fs := newFileSet(t, folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) 858 859 // Add files such that we are in sync for a and b, and need c and d. 860 files := []protocol.FileInfo{ 861 {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, 862 {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, 863 {Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, 864 {Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, 865 } 866 fs.Update(protocol.LocalDeviceID, files) 867 files[2].Version = files[2].Version.Update(remoteDevice0.Short()) 868 files[3].Version = files[2].Version.Update(remoteDevice0.Short()) 869 fs.Update(remoteDevice0, files) 870 871 checkNeed := func() { 872 snap := snapshot(t, fs) 873 defer snap.Release() 874 c := snap.NeedSize(protocol.LocalDeviceID) 875 if c.Files != 2 { 876 t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files) 877 } 878 needed := make([]protocol.FileInfo, 0, 2) 879 snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { 880 needed = append(needed, fi.(protocol.FileInfo)) 881 return true 882 }) 883 if l := len(needed); l != 2 { 884 t.Errorf("Expected 2 needed files locally, got %v in db", l) 885 } else if needed[0].Name != "c" || needed[1].Name != "d" { 886 t.Errorf("Expected files c and d to be needed, got %v and %v", needed[0].Name, needed[1].Name) 887 } 888 } 889 890 checkNeed() 891 892 trans, err := db.newReadWriteTransaction() 893 if err != nil { 894 t.Fatal(err) 895 } 896 defer trans.close() 897 898 // Add "b" to needed and remove "d" 899 folder := []byte(folderStr) 900 key, err := trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[1].Name)) 901 if err != nil { 902 t.Fatal(err) 903 } 904 if err = trans.Put(key, nil); err != nil { 905 t.Fatal(err) 906 } 907 key, err = trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[3].Name)) 908 if err != nil { 909 t.Fatal(err) 910 } 911 if err = trans.Delete(key); err != nil { 912 t.Fatal(err) 913 } 914 if err := trans.Commit(); err != nil { 915 t.Fatal(err) 916 } 917 918 if repaired, err := db.checkLocalNeed(folder); err != nil { 919 t.Fatal(err) 920 } else if repaired != 2 { 921 t.Error("Expected 2 repaired local need items, got", repaired) 922 } 923 924 checkNeed() 925} 926 927func TestDuplicateNeedCount(t *testing.T) { 928 db := newLowlevelMemory(t) 929 defer db.Close() 930 931 folder := "test" 932 testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "") 933 934 fs := newFileSet(t, folder, testFs, db) 935 files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}} 936 fs.Update(protocol.LocalDeviceID, files) 937 files[0].Version = files[0].Version.Update(remoteDevice0.Short()) 938 fs.Update(remoteDevice0, files) 939 940 db.checkRepair() 941 942 fs = newFileSet(t, folder, testFs, db) 943 found := false 944 for _, c := range fs.meta.counts.Counts { 945 if bytes.Equal(protocol.LocalDeviceID[:], c.DeviceID) && c.LocalFlags == needFlag { 946 if found { 947 t.Fatal("second need count for local device encountered") 948 } 949 found = true 950 } 951 } 952 if !found { 953 t.Fatal("no need count for local device encountered") 954 } 955} 956 957func TestNeedAfterDropGlobal(t *testing.T) { 958 db := newLowlevelMemory(t) 959 defer db.Close() 960 961 folder := "test" 962 testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "") 963 964 fs := newFileSet(t, folder, testFs, db) 965 966 // Initial: 967 // Three devices and a file "test": local has Version 1, remoteDevice0 968 // Version 2 and remoteDevice2 doesn't have it. 969 // All of them have "bar", just so the db knows about remoteDevice2. 970 files := []protocol.FileInfo{ 971 {Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}, 972 {Name: "bar", Version: protocol.Vector{}.Update(myID), Sequence: 2}, 973 } 974 fs.Update(protocol.LocalDeviceID, files) 975 files[0].Version = files[0].Version.Update(myID) 976 fs.Update(remoteDevice0, files) 977 fs.Update(remoteDevice1, files[1:]) 978 979 // remoteDevice1 needs one file: test 980 snap := snapshot(t, fs) 981 c := snap.NeedSize(remoteDevice1) 982 if c.Files != 1 { 983 t.Errorf("Expected 1 needed files initially, got %v", c.Files) 984 } 985 snap.Release() 986 987 // Drop remoteDevice0, i.e. remove all their files from db. 988 // That changes the global file, which is now what local has. 989 fs.Drop(remoteDevice0) 990 991 // remoteDevice1 still needs test. 992 snap = snapshot(t, fs) 993 c = snap.NeedSize(remoteDevice1) 994 if c.Files != 1 { 995 t.Errorf("Expected still 1 needed files, got %v", c.Files) 996 } 997 snap.Release() 998} 999 1000func numBlockLists(db *Lowlevel) (int, error) { 1001 it, err := db.Backend.NewPrefixIterator([]byte{KeyTypeBlockList}) 1002 if err != nil { 1003 return 0, err 1004 } 1005 defer it.Release() 1006 n := 0 1007 for it.Next() { 1008 n++ 1009 } 1010 if err := it.Error(); err != nil { 1011 return 0, err 1012 } 1013 return n, nil 1014} 1015