1// Copyright 2016 Keybase Inc. All rights reserved. 2// Use of this source code is governed by a BSD 3// license that can be found in the LICENSE file. 4 5package libkbfs 6 7import ( 8 "fmt" 9 "os" 10 "sync" 11 "testing" 12 "time" 13 14 "github.com/keybase/client/go/kbfs/data" 15 "github.com/keybase/client/go/kbfs/ioutil" 16 "github.com/keybase/client/go/kbfs/kbfsmd" 17 "github.com/keybase/client/go/kbfs/libcontext" 18 "github.com/keybase/client/go/kbfs/test/clocktest" 19 "github.com/keybase/client/go/kbfs/tlf" 20 "github.com/keybase/client/go/kbfs/tlfhandle" 21 kbname "github.com/keybase/client/go/kbun" 22 "github.com/stretchr/testify/assert" 23 "github.com/stretchr/testify/require" 24 "golang.org/x/net/context" 25) 26 27func readAndCompareData(ctx context.Context, t *testing.T, config Config, 28 name string, expectedData []byte, user kbname.NormalizedUsername) { 29 rootNode := GetRootNodeOrBust(ctx, t, config, name, tlf.Private) 30 31 kbfsOps := config.KBFSOps() 32 fileNode, _, err := kbfsOps.Lookup(ctx, rootNode, testPPS("a")) 33 require.NoError(t, err) 34 data := make([]byte, 1) 35 _, err = kbfsOps.Read(ctx, fileNode, data, 0) 36 require.NoError(t, err) 37 assert.Equal(t, expectedData[0], data[0]) 38} 39 40type testCRObserver struct { 41 c chan<- struct{} 42 changes []NodeChange 43} 44 45func (t *testCRObserver) LocalChange(ctx context.Context, node Node, 46 write WriteRange) { 47 // ignore 48} 49 50func (t *testCRObserver) BatchChanges(ctx context.Context, 51 changes []NodeChange, _ []NodeID) { 52 t.changes = append(t.changes, changes...) 53 if len(changes) > 0 { 54 t.c <- struct{}{} 55 } 56} 57 58func (t *testCRObserver) TlfHandleChange(ctx context.Context, 59 newHandle *tlfhandle.Handle) { 60} 61 62func checkStatus(ctx context.Context, t *testing.T, kbfsOps KBFSOps, 63 staged bool, headWriter kbname.NormalizedUsername, dirtyPaths []string, fb data.FolderBranch, 64 prefix string) { 65 status, _, err := kbfsOps.FolderStatus(ctx, fb) 66 require.NoError(t, err) 67 assert.Equal(t, status.Staged, staged) 68 assert.Equal(t, status.HeadWriter, headWriter) 69 checkStringSlices(t, dirtyPaths, status.DirtyPaths) 70} 71 72func TestBasicMDUpdate(t *testing.T) { 73 // simulate two users 74 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 75 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 76 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 77 78 config2 := ConfigAsUser(config1, userName2) 79 defer CheckConfigAndShutdown(ctx, t, config2) 80 81 name := userName1.String() + "," + userName2.String() 82 83 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 84 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 85 86 kbfsOps2 := config2.KBFSOps() 87 _, statusChan, err := kbfsOps2.FolderStatus(ctx, rootNode2.GetFolderBranch()) 88 require.NoError(t, err) 89 90 // user 1 creates a file 91 kbfsOps1 := config1.KBFSOps() 92 _, _, err = kbfsOps1.CreateFile(ctx, rootNode1, testPPS("a"), false, NoExcl) 93 require.NoError(t, err) 94 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 95 require.NoError(t, err) 96 97 err = kbfsOps2.SyncFromServer(ctx, 98 rootNode2.GetFolderBranch(), nil) 99 require.NoError(t, err) 100 101 entries, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 102 require.NoError(t, err) 103 require.Equal(t, 1, len(entries)) 104 _, ok := entries[rootNode2.ChildName("a")] 105 require.True(t, ok) 106 107 // The status should have fired as well (though in this case the 108 // writer is the same as before) 109 <-statusChan 110 checkStatus(ctx, t, kbfsOps1, false, userName1, nil, 111 rootNode1.GetFolderBranch(), "Node 1") 112 checkStatus(ctx, t, kbfsOps2, false, userName1, nil, 113 rootNode2.GetFolderBranch(), "Node 2") 114} 115 116func testMultipleMDUpdates(t *testing.T, unembedChanges bool) { 117 // simulate two users 118 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 119 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 120 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 121 122 config2 := ConfigAsUser(config1, userName2) 123 defer CheckConfigAndShutdown(ctx, t, config2) 124 125 if unembedChanges { 126 bss1, ok1 := config1.BlockSplitter().(*data.BlockSplitterSimple) 127 require.True(t, ok1) 128 bss2, ok2 := config2.BlockSplitter().(*data.BlockSplitterSimple) 129 require.True(t, ok2) 130 bss1.SetBlockChangeEmbedMaxSizeForTesting(3) 131 bss2.SetBlockChangeEmbedMaxSizeForTesting(3) 132 } 133 134 name := userName1.String() + "," + userName2.String() 135 136 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 137 138 kbfsOps1 := config1.KBFSOps() 139 // user 1 creates a file 140 _, _, err := kbfsOps1.CreateFile( 141 ctx, rootNode1, testPPS("a"), false, NoExcl) 142 require.NoError(t, err) 143 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 144 require.NoError(t, err) 145 146 // user 2 looks up the directory (and sees the file) 147 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 148 149 // now user 1 renames the old file, and creates a new one 150 err = kbfsOps1.Rename(ctx, rootNode1, testPPS("a"), rootNode1, testPPS("b")) 151 require.NoError(t, err) 152 _, _, err = kbfsOps1.CreateFile(ctx, rootNode1, testPPS("c"), false, NoExcl) 153 require.NoError(t, err) 154 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 155 require.NoError(t, err) 156 157 kbfsOps2 := config2.KBFSOps() 158 err = kbfsOps2.SyncFromServer(ctx, 159 rootNode2.GetFolderBranch(), nil) 160 require.NoError(t, err) 161 162 entries, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 163 require.NoError(t, err) 164 require.Equal(t, 2, len(entries)) 165 _, ok := entries[rootNode2.ChildName("b")] 166 require.True(t, ok) 167 _, ok = entries[rootNode2.ChildName("c")] 168 require.True(t, ok) 169} 170 171func TestMultipleMDUpdates(t *testing.T) { 172 testMultipleMDUpdates(t, false) 173} 174 175func TestMultipleMDUpdatesUnembedChanges(t *testing.T) { 176 testMultipleMDUpdates(t, true) 177} 178 179func TestGetTLFCryptKeysWhileUnmergedAfterRestart(t *testing.T) { 180 tempdir, err := ioutil.TempDir(os.TempDir(), "journal_for_gettlfcryptkeys") 181 require.NoError(t, err) 182 defer func() { 183 err := ioutil.RemoveAll(tempdir) 184 assert.NoError(t, err) 185 }() 186 187 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 188 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 189 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 190 191 // enable journaling to see patrick's error 192 err = config1.EnableDiskLimiter(tempdir) 193 require.NoError(t, err) 194 err = config1.EnableJournaling( 195 ctx, tempdir, TLFJournalBackgroundWorkEnabled) 196 require.NoError(t, err) 197 jManager, err := GetJournalManager(config1) 198 require.NoError(t, err) 199 jManager.onBranchChange = nil 200 jManager.onMDFlush = nil 201 err = jManager.EnableAuto(ctx) 202 require.NoError(t, err) 203 204 config2 := ConfigAsUser(config1, userName2) 205 defer CheckConfigAndShutdown(ctx, t, config2) 206 name := userName1.String() + "," + userName2.String() 207 208 // user1 creates a file in a shared dir 209 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 210 211 kbfsOps1 := config1.KBFSOps() 212 fileNode1, _, err := kbfsOps1.CreateFile( 213 ctx, rootNode1, testPPS("a"), false, NoExcl) 214 require.NoError(t, err) 215 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 216 require.NoError(t, err) 217 218 _, err = DisableUpdatesForTesting(config1, rootNode1.GetFolderBranch()) 219 require.NoError(t, err) 220 err = DisableCRForTesting(config1, rootNode1.GetFolderBranch()) 221 require.NoError(t, err) 222 223 // Wait for "a" to flush to the server. 224 err = jManager.Wait(ctx, rootNode1.GetFolderBranch().Tlf) 225 require.NoError(t, err) 226 227 // then user2 write to the file 228 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 229 230 kbfsOps2 := config2.KBFSOps() 231 fileNode2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 232 require.NoError(t, err) 233 data2 := []byte{2} 234 err = kbfsOps2.Write(ctx, fileNode2, data2, 0) 235 require.NoError(t, err) 236 checkStatus(ctx, t, kbfsOps2, false, userName1, []string{"u1,u2/a"}, 237 rootNode2.GetFolderBranch(), "Node 2 (after write)") 238 err = kbfsOps2.SyncAll(ctx, fileNode2.GetFolderBranch()) 239 require.NoError(t, err) 240 241 // Now when user 1 tries to write to file 1 and sync, it will 242 // become unmerged. 243 data1 := []byte{1} 244 err = kbfsOps1.Write(ctx, fileNode1, data1, 0) 245 require.NoError(t, err) 246 // sync the file from u1 so that we get a clean exit state 247 err = kbfsOps1.SyncAll(ctx, fileNode1.GetFolderBranch()) 248 require.NoError(t, err) 249 250 // Wait for the conflict to be detected. 251 err = jManager.Wait(ctx, rootNode1.GetFolderBranch().Tlf) 252 require.NoError(t, err) 253 254 // now re-login u1 255 config1B := ConfigAsUser(config1, userName1) 256 err = config1B.EnableDiskLimiter(tempdir) 257 require.NoError(t, err) 258 defer CheckConfigAndShutdown(ctx, t, config1B) 259 err = config1B.EnableJournaling( 260 ctx, tempdir, TLFJournalBackgroundWorkEnabled) 261 require.NoError(t, err) 262 jManager, err = GetJournalManager(config1B) 263 require.NoError(t, err) 264 jManager.onBranchChange = nil 265 jManager.onMDFlush = nil 266 267 err = DisableCRForTesting(config1B, rootNode1.GetFolderBranch()) 268 require.NoError(t, err) 269 270 tlfHandle, err := tlfhandle.ParseHandle( 271 ctx, config1B.KBPKI(), config1B.MDOps(), nil, name, tlf.Private) 272 require.NoError(t, err) 273 274 _, _, err = config1B.KBFSOps().GetTLFCryptKeys(ctx, tlfHandle) 275 require.NoError(t, err) 276} 277 278// Tests that, in the face of a conflict, a user will commit its 279// changes to a private branch, which will persist after restart (and 280// the other user will be unaffected). 281func TestUnmergedAfterRestart(t *testing.T) { 282 // simulate two users 283 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 284 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 285 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 286 287 config2 := ConfigAsUser(config1, userName2) 288 defer CheckConfigAndShutdown(ctx, t, config2) 289 290 name := userName1.String() + "," + userName2.String() 291 292 // user1 creates a file in a shared dir 293 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 294 295 kbfsOps1 := config1.KBFSOps() 296 fileNode1, _, err := kbfsOps1.CreateFile( 297 ctx, rootNode1, testPPS("a"), false, NoExcl) 298 require.NoError(t, err) 299 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 300 require.NoError(t, err) 301 302 _, err = DisableUpdatesForTesting(config1, rootNode1.GetFolderBranch()) 303 require.NoError(t, err) 304 err = DisableCRForTesting(config1, rootNode1.GetFolderBranch()) 305 require.NoError(t, err) 306 307 // then user2 write to the file 308 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 309 310 kbfsOps2 := config2.KBFSOps() 311 fileNode2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 312 require.NoError(t, err) 313 data2 := []byte{2} 314 err = kbfsOps2.Write(ctx, fileNode2, data2, 0) 315 require.NoError(t, err) 316 checkStatus(ctx, t, kbfsOps2, false, userName1, []string{"u1,u2/a"}, 317 rootNode2.GetFolderBranch(), "Node 2 (after write)") 318 err = kbfsOps2.SyncAll(ctx, fileNode2.GetFolderBranch()) 319 require.NoError(t, err) 320 321 // Now when user 1 tries to write to file 1 and sync, it will 322 // become unmerged. Because this happens in the same goroutine as 323 // the above Sync, we can be sure that the updater on client 1 324 // hasn't yet seen the MD update, and so its Sync will present a 325 // conflict. 326 data1 := []byte{1} 327 err = kbfsOps1.Write(ctx, fileNode1, data1, 0) 328 require.NoError(t, err) 329 checkStatus(ctx, t, kbfsOps1, false, userName1, []string{"u1,u2/a"}, 330 rootNode1.GetFolderBranch(), "Node 1 (after write)") 331 err = kbfsOps1.SyncAll(ctx, fileNode1.GetFolderBranch()) 332 require.NoError(t, err) 333 334 checkStatus(ctx, t, kbfsOps1, true, userName1, nil, 335 rootNode1.GetFolderBranch(), "Node 1") 336 checkStatus(ctx, t, kbfsOps2, false, userName2, nil, 337 rootNode2.GetFolderBranch(), "Node 2") 338 339 // now re-login the users, and make sure 1 can see the changes, 340 // but 2 can't 341 config1B := ConfigAsUser(config1, userName1) 342 defer CheckConfigAndShutdown(ctx, t, config1B) 343 config2B := ConfigAsUser(config1, userName2) 344 defer CheckConfigAndShutdown(ctx, t, config2B) 345 346 err = DisableCRForTesting(config1B, rootNode1.GetFolderBranch()) 347 require.NoError(t, err) 348 349 // Keep the config1B node in memory, so it doesn't get garbage 350 // collected (preventing notifications) 351 rootNode1B := GetRootNodeOrBust(ctx, t, config1B, name, tlf.Private) 352 353 kbfsOps1B := config1B.KBFSOps() 354 fileNode1B, _, err := kbfsOps1B.Lookup(ctx, rootNode1B, testPPS("a")) 355 require.NoError(t, err) 356 357 readAndCompareData(ctx, t, config1B, name, data1, userName1) 358 readAndCompareData(ctx, t, config2B, name, data2, userName2) 359 360 checkStatus(ctx, t, config1B.KBFSOps(), true, userName1, nil, 361 fileNode1B.GetFolderBranch(), "Node 1") 362 checkStatus(ctx, t, config2B.KBFSOps(), false, userName2, nil, 363 rootNode2.GetFolderBranch(), "Node 2") 364 365 // register as a listener before the unstaging happens 366 c := make(chan struct{}, 2) 367 cro := &testCRObserver{c, nil} 368 err = config1B.Notifier().RegisterForChanges( 369 []data.FolderBranch{rootNode1B.GetFolderBranch()}, cro) 370 require.NoError(t, err) 371 372 ops1B := getOps(config1B, fileNode1B.GetFolderBranch().Tlf) 373 ops2B := getOps(config2B, fileNode1B.GetFolderBranch().Tlf) 374 lState := makeFBOLockState() 375 require.Equal(t, ops1B.getLatestMergedRevision(lState), ops2B.getCurrMDRevision(lState)) 376 377 // Unstage user 1's changes, and make sure everyone is back in 378 // sync. TODO: remove this once we have automatic conflict 379 // resolution. 380 err = config1B.KBFSOps().UnstageForTesting(ctx, 381 rootNode1B.GetFolderBranch()) 382 require.NoError(t, err) 383 384 // we should have had at least two updates, one for the unstaging and one 385 // for the fast-forward 386 select { 387 case <-c: 388 default: 389 t.Fatal("No update!") 390 } 391 select { 392 case <-c: 393 default: 394 t.Fatal("No 2nd update!") 395 } 396 // make sure we see two sync op changes, on the same node 397 assert.Equal(t, 2, len(cro.changes)) 398 var n Node 399 for _, change := range cro.changes { 400 if n == nil { 401 n = change.Node 402 } else { 403 assert.Equal(t, n.GetID(), change.Node.GetID()) 404 } 405 } 406 407 err = config1B.KBFSOps().SyncFromServer( 408 ctx, fileNode1B.GetFolderBranch(), nil) 409 require.NoError(t, err) 410 err = config2B.KBFSOps(). 411 SyncFromServer(ctx, 412 rootNode2.GetFolderBranch(), nil) 413 require.NoError(t, err) 414 415 readAndCompareData(ctx, t, config1B, name, data2, userName2) 416 readAndCompareData(ctx, t, config2B, name, data2, userName2) 417 checkStatus(ctx, t, config1B.KBFSOps(), false, userName1, nil, 418 rootNode1.GetFolderBranch(), "Node 1 (after unstage)") 419 checkStatus(ctx, t, config2B.KBFSOps(), false, userName1, nil, 420 rootNode2.GetFolderBranch(), "Node 2 (after unstage)") 421} 422 423// Tests that multiple users can write to the same file sequentially 424// without any problems. 425func TestMultiUserWrite(t *testing.T) { 426 // simulate two users 427 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 428 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 429 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 430 431 config2 := ConfigAsUser(config1, userName2) 432 defer CheckConfigAndShutdown(ctx, t, config2) 433 434 name := userName1.String() + "," + userName2.String() 435 436 // user1 creates a file in a shared dir 437 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 438 439 kbfsOps1 := config1.KBFSOps() 440 _, _, err := kbfsOps1.CreateFile( 441 ctx, rootNode1, testPPS("a"), false, NoExcl) 442 require.NoError(t, err) 443 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 444 require.NoError(t, err) 445 446 // then user2 write to the file 447 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 448 449 kbfsOps2 := config2.KBFSOps() 450 fileNode2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 451 require.NoError(t, err) 452 453 data2 := []byte{2} 454 err = kbfsOps2.Write(ctx, fileNode2, data2, 0) 455 require.NoError(t, err) 456 // Write twice to make sure that multiple write operations within 457 // a sync work when the writer is changing. 458 err = kbfsOps2.Write(ctx, fileNode2, data2, 0) 459 require.NoError(t, err) 460 err = kbfsOps2.SyncAll(ctx, fileNode2.GetFolderBranch()) 461 require.NoError(t, err) 462 readAndCompareData(ctx, t, config2, name, data2, userName2) 463 464 // A second write by the same user 465 data3 := []byte{3} 466 err = kbfsOps2.Write(ctx, fileNode2, data3, 0) 467 require.NoError(t, err) 468 err = kbfsOps2.SyncAll(ctx, fileNode2.GetFolderBranch()) 469 require.NoError(t, err) 470 471 readAndCompareData(ctx, t, config2, name, data3, userName2) 472 473 err = kbfsOps1.SyncFromServer(ctx, 474 rootNode1.GetFolderBranch(), nil) 475 require.NoError(t, err) 476 readAndCompareData(ctx, t, config1, name, data3, userName2) 477} 478 479func testBasicCRNoConflict(t *testing.T, unembedChanges bool) { 480 // simulate two users 481 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 482 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 483 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 484 485 config2 := ConfigAsUser(config1, userName2) 486 defer CheckConfigAndShutdown(ctx, t, config2) 487 488 if unembedChanges { 489 bss1, ok1 := config1.BlockSplitter().(*data.BlockSplitterSimple) 490 require.True(t, ok1) 491 bss2, ok2 := config2.BlockSplitter().(*data.BlockSplitterSimple) 492 require.True(t, ok2) 493 // 128 seems to be a good size that works on both 386 and x64 494 // platforms. 495 bss1.SetBlockChangeEmbedMaxSizeForTesting(128) 496 bss2.SetBlockChangeEmbedMaxSizeForTesting(128) 497 } 498 499 name := userName1.String() + "," + userName2.String() 500 501 // user1 creates a file in a shared dir 502 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 503 504 kbfsOps1 := config1.KBFSOps() 505 _, _, err := kbfsOps1.CreateFile( 506 ctx, rootNode1, testPPS("a"), false, NoExcl) 507 require.NoError(t, err) 508 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 509 require.NoError(t, err) 510 511 // look it up on user2 512 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 513 514 kbfsOps2 := config2.KBFSOps() 515 _, _, err = kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 516 require.NoError(t, err) 517 518 // disable updates on user 2 519 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 520 require.NoError(t, err) 521 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 522 require.NoError(t, err) 523 524 // User 1 makes a new file 525 _, _, err = kbfsOps1.CreateFile( 526 ctx, rootNode1, testPPS("b"), false, NoExcl) 527 require.NoError(t, err) 528 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 529 require.NoError(t, err) 530 531 // User 2 makes a new different file 532 _, _, err = kbfsOps2.CreateFile( 533 ctx, rootNode2, testPPS("c"), false, NoExcl) 534 require.NoError(t, err) 535 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 536 require.NoError(t, err) 537 538 // re-enable updates, and wait for CR to complete 539 c <- struct{}{} 540 err = RestartCRForTesting( 541 libcontext.BackgroundContextWithCancellationDelayer(), config2, 542 rootNode2.GetFolderBranch()) 543 require.NoError(t, err) 544 err = kbfsOps2.SyncFromServer(ctx, 545 rootNode2.GetFolderBranch(), nil) 546 require.NoError(t, err) 547 548 err = kbfsOps1.SyncFromServer(ctx, 549 rootNode1.GetFolderBranch(), nil) 550 require.NoError(t, err) 551 552 // Make sure they both see the same set of children 553 expectedChildren := []string{"a", "b", "c"} 554 children1, err := kbfsOps1.GetDirChildren(ctx, rootNode1) 555 require.NoError(t, err) 556 557 children2, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 558 require.NoError(t, err) 559 560 assert.Equal(t, len(expectedChildren), len(children1)) 561 562 for _, child := range expectedChildren { 563 _, ok := children1[rootNode1.ChildName(child)] 564 assert.True(t, ok) 565 } 566 567 require.Equal(t, children1, children2) 568 569 if unembedChanges { 570 // Make sure the MD has an unembedded change block. 571 md, err := config1.MDOps().GetForTLF(ctx, 572 rootNode1.GetFolderBranch().Tlf, nil) 573 require.NoError(t, err) 574 require.NotEqual(t, data.ZeroPtr, md.data.cachedChanges.Info.BlockPointer) 575 } 576} 577 578// Tests that two users can make independent writes while forked, and 579// conflict resolution will merge them correctly. 580func TestBasicCRNoConflict(t *testing.T) { 581 testBasicCRNoConflict(t, false) 582} 583 584// Tests same as above, with unembedded block changes 585func TestBasicCRNoConflictWithUnembeddedBlockChanges(t *testing.T) { 586 testBasicCRNoConflict(t, true) 587} 588 589type registerForUpdateRecord struct { 590 id tlf.ID 591 currHead kbfsmd.Revision 592} 593 594type mdServerLocalRecordingRegisterForUpdate struct { 595 mdServerLocal 596 ch chan<- registerForUpdateRecord 597} 598 599// newMDServerLocalRecordingRegisterForUpdate returns a wrapper of 600// MDServerLocal that records RegisterforUpdate calls. 601func newMDServerLocalRecordingRegisterForUpdate(mdServerRaw mdServerLocal) ( 602 mdServer mdServerLocalRecordingRegisterForUpdate, 603 records <-chan registerForUpdateRecord) { 604 ch := make(chan registerForUpdateRecord, 8) 605 ret := mdServerLocalRecordingRegisterForUpdate{mdServerRaw, ch} 606 return ret, ch 607} 608 609func (md mdServerLocalRecordingRegisterForUpdate) RegisterForUpdate( 610 ctx context.Context, 611 id tlf.ID, currHead kbfsmd.Revision) (<-chan error, error) { 612 md.ch <- registerForUpdateRecord{id: id, currHead: currHead} 613 return md.mdServerLocal.RegisterForUpdate(ctx, id, currHead) 614} 615 616func TestCRFileConflictWithMoreUpdatesFromOneUser(t *testing.T) { 617 // simulate two users 618 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 619 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 620 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 621 622 config2 := ConfigAsUser(config1, userName2) 623 mdServ, chForMdServer2 := newMDServerLocalRecordingRegisterForUpdate( 624 config2.MDServer().(mdServerLocal)) 625 config2.SetMDServer(mdServ) 626 defer CheckConfigAndShutdown(ctx, t, config2) 627 628 name := userName1.String() + "," + userName2.String() 629 630 // user1 creates a file in a shared dir 631 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 632 633 kbfsOps1 := config1.KBFSOps() 634 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 635 require.NoError(t, err) 636 fileB1, _, err := kbfsOps1.CreateFile(ctx, dirA1, testPPS("b"), false, NoExcl) 637 require.NoError(t, err) 638 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 639 require.NoError(t, err) 640 641 // look it up on user2 642 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 643 644 kbfsOps2 := config2.KBFSOps() 645 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 646 require.NoError(t, err) 647 fileB2, _, err := kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 648 require.NoError(t, err) 649 650 // disable updates on user 2 651 chForEnablingUpdates, err := DisableUpdatesForTesting( 652 config2, rootNode2.GetFolderBranch()) 653 require.NoError(t, err) 654 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 655 require.NoError(t, err) 656 657 // User 1 writes the file 658 data := []byte{1, 2, 3, 4, 5} 659 err = kbfsOps1.Write(ctx, fileB1, data, 0) 660 require.NoError(t, err) 661 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 662 require.NoError(t, err) 663 664 // User 2 makes a few changes in the file 665 for i := byte(0); i < 4; i++ { 666 // This makes sure the unmerged head of user 2 is ahead of (has large 667 // revision than) the merged master branch, so we can test that we properly 668 // fetch updates when unmerged revision number is greater than merged 669 // revision number (regression in KBFS-1206). 670 671 data = []byte{1, 2, 3, 4, i} 672 err = kbfsOps2.Write(ctx, fileB2, data, 0) 673 require.NoError(t, err) 674 675 err = kbfsOps2.SyncAll(ctx, fileB2.GetFolderBranch()) 676 require.NoError(t, err) 677 } 678 679 chForEnablingUpdates <- struct{}{} 680 681 equal := false 682 683 // check for at most 4 times. This should be sufficiently long for client get 684 // latest merged revision and register with that 685 for i := 0; i < 4; i++ { 686 record := <-chForMdServer2 687 mergedRev, err := mdServ.getCurrentMergedHeadRevision( 688 ctx, rootNode2.GetFolderBranch().Tlf) 689 require.NoError(t, err) 690 if record.currHead == mergedRev { 691 equal = true 692 break 693 } 694 } 695 696 require.True(t, equal) 697} 698 699// Tests that two users can make independent writes while forked, and 700// conflict resolution will merge them correctly. 701func TestBasicCRFileConflict(t *testing.T) { 702 // simulate two users 703 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 704 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 705 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 706 707 config2 := ConfigAsUser(config1, userName2) 708 defer CheckConfigAndShutdown(ctx, t, config2) 709 710 clock, now := clocktest.NewTestClockAndTimeNow() 711 config2.SetClock(clock) 712 713 name := userName1.String() + "," + userName2.String() 714 715 // user1 creates a file in a shared dir 716 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 717 718 kbfsOps1 := config1.KBFSOps() 719 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 720 require.NoError(t, err) 721 fileB1, _, err := kbfsOps1.CreateFile( 722 ctx, dirA1, testPPS("b"), false, NoExcl) 723 require.NoError(t, err) 724 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 725 require.NoError(t, err) 726 727 // look it up on user2 728 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 729 730 kbfsOps2 := config2.KBFSOps() 731 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 732 require.NoError(t, err) 733 fileB2, _, err := kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 734 require.NoError(t, err) 735 736 // disable updates on user 2 737 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 738 require.NoError(t, err) 739 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 740 require.NoError(t, err) 741 742 // User 1 writes the file 743 data1 := []byte{1, 2, 3, 4, 5} 744 err = kbfsOps1.Write(ctx, fileB1, data1, 0) 745 require.NoError(t, err) 746 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 747 require.NoError(t, err) 748 749 // User 2 makes a new different file 750 data2 := []byte{5, 4, 3, 2, 1} 751 err = kbfsOps2.Write(ctx, fileB2, data2, 0) 752 require.NoError(t, err) 753 err = kbfsOps2.SyncAll(ctx, fileB2.GetFolderBranch()) 754 require.NoError(t, err) 755 756 // re-enable updates, and wait for CR to complete 757 c <- struct{}{} 758 err = RestartCRForTesting( 759 libcontext.BackgroundContextWithCancellationDelayer(), config2, 760 rootNode2.GetFolderBranch()) 761 require.NoError(t, err) 762 err = kbfsOps2.SyncFromServer(ctx, 763 rootNode2.GetFolderBranch(), nil) 764 require.NoError(t, err) 765 766 err = kbfsOps1.SyncFromServer(ctx, 767 rootNode1.GetFolderBranch(), nil) 768 require.NoError(t, err) 769 770 cre := WriterDeviceDateConflictRenamer{} 771 // Make sure they both see the same set of children 772 expectedChildren := []string{ 773 "b", 774 cre.ConflictRenameHelper(now, "u2", "dev1", "b"), 775 } 776 children1, err := kbfsOps1.GetDirChildren(ctx, dirA1) 777 require.NoError(t, err) 778 779 children2, err := kbfsOps2.GetDirChildren(ctx, dirA2) 780 require.NoError(t, err) 781 782 assert.Equal(t, len(expectedChildren), len(children1)) 783 784 for _, child := range expectedChildren { 785 _, ok := children1[dirA1.ChildName(child)] 786 assert.True(t, ok) 787 } 788 789 require.Equal(t, children1, children2) 790} 791 792// Tests that if CR fails enough times it will stop trying, 793// and that we can move the conflicts out of the way. 794func TestBasicCRFailureAndFixing(t *testing.T) { 795 tempdir, err := ioutil.TempDir(os.TempDir(), "journal_for_fail_fix") 796 defer os.RemoveAll(tempdir) 797 798 // simulate two users 799 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 800 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 801 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 802 803 config2 := ConfigAsUser(config1, userName2) 804 defer CheckConfigAndShutdown(ctx, t, config2) 805 806 // Enable journaling on user 2 807 require.NoError(t, err) 808 err = config2.EnableDiskLimiter(tempdir) 809 require.NoError(t, err) 810 err = config2.EnableJournaling(ctx, tempdir, 811 TLFJournalBackgroundWorkEnabled) 812 require.NoError(t, err) 813 jManager, err := GetJournalManager(config2) 814 require.NoError(t, err) 815 err = jManager.EnableAuto(ctx) 816 require.NoError(t, err) 817 818 clock, now := clocktest.NewTestClockAndTimeNow() 819 config2.SetClock(clock) 820 821 name := userName1.String() + "," + userName2.String() 822 823 t.Log("User 1 creates a file a/b.") 824 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 825 826 kbfsOps1 := config1.KBFSOps() 827 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 828 require.NoError(t, err) 829 fileB1, _, err := kbfsOps1.CreateFile( 830 ctx, dirA1, testPPS("b"), false, NoExcl) 831 require.NoError(t, err) 832 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 833 require.NoError(t, err) 834 835 t.Log("User 2 looks up file a/b.") 836 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 837 838 kbfsOps2 := config2.KBFSOps() 839 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 840 require.NoError(t, err) 841 fileB2, _, err := kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 842 require.NoError(t, err) 843 844 err = SetCRFailureForTesting(ctx, config2, rootNode2.GetFolderBranch(), 845 alwaysFailCR) 846 require.NoError(t, err) 847 848 t.Log("Disable updates on user 2.") 849 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 850 require.NoError(t, err) 851 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 852 require.NoError(t, err) 853 854 t.Log("User 1 writes to file a/b.") 855 data1 := []byte{1, 2, 3, 4, 5} 856 err = kbfsOps1.Write(ctx, fileB1, data1, 0) 857 require.NoError(t, err) 858 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 859 require.NoError(t, err) 860 861 t.Log("User 2 writes to file a/b without having heard user 1's update.") 862 data2 := []byte{5, 4, 3, 2, 1} 863 err = kbfsOps2.Write(ctx, fileB2, data2, 0) 864 require.NoError(t, err) 865 err = kbfsOps2.SyncAll(ctx, fileB2.GetFolderBranch()) 866 require.NoError(t, err) 867 868 t.Log("Reenable updates and wait for CR to fail.") 869 c <- struct{}{} 870 err = RestartCRForTesting( 871 libcontext.BackgroundContextWithCancellationDelayer(), config2, 872 rootNode2.GetFolderBranch()) 873 require.NoError(t, err) 874 875 t.Log("Try to SyncFromServer on user 2.") 876 err = kbfsOps2.SyncFromServer(ctx, 877 rootNode2.GetFolderBranch(), nil) 878 require.Equal(t, &ErrStillStagedAfterCR{}, err) 879 880 ops, ok := config2.KBFSOps().(*KBFSOpsStandard) 881 require.True(t, ok) 882 fbo := ops.getOpsNoAdd(ctx, rootNode2.GetFolderBranch()) 883 884 t.Log("Write a bunch more files as user 2, creating more conflicts.") 885 for i := 0; i < maxConflictResolutionAttempts; i++ { 886 fileName := fmt.Sprintf("file%d", i) 887 newFile, _, err := kbfsOps2.CreateFile( 888 ctx, dirA2, testPPS(fileName), false, NoExcl) 889 require.NoError(t, err, "Loop %d", i) 890 err = kbfsOps2.SyncAll(ctx, newFile.GetFolderBranch()) 891 require.NoError(t, err, "Loop %d", i) 892 err = fbo.cr.Wait(ctx) 893 require.NoError(t, err, "Loop %d", i) 894 } 895 896 t.Log("Check that there is conflict state in the CR DB.") 897 crdb := config2.GetConflictResolutionDB() 898 crData, err := crdb.Get(fbo.id().Bytes(), nil) 899 require.NoError(t, err) 900 require.NotZero(t, len(crData)) 901 902 t.Log("Clear the conflict state and re-enable CR.") 903 err = fbo.clearConflictView(ctx) 904 require.NoError(t, err) 905 906 err = SetCRFailureForTesting(ctx, config2, rootNode2.GetFolderBranch(), 907 doNotAlwaysFailCR) 908 require.NoError(t, err) 909 910 t.Log("Trigger CR and wait for it to resolve.") 911 dirA2, _, err = kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 912 require.NoError(t, err) 913 _, _, err = kbfsOps2.CreateFile( 914 ctx, dirA2, testPPS("newFile"), false, NoExcl) 915 require.NoError(t, err) 916 917 err = kbfsOps2.SyncAll(ctx, dirA2.GetFolderBranch()) 918 require.NoError(t, err) 919 err = fbo.cr.Wait(ctx) 920 require.NoError(t, err) 921 922 t.Log("Verify that the conflict is resolved.") 923 err = kbfsOps2.SyncFromServer(ctx, 924 rootNode2.GetFolderBranch(), nil) 925 require.NoError(t, err) 926 927 err = kbfsOps1.SyncFromServer(ctx, 928 rootNode1.GetFolderBranch(), nil) 929 require.NoError(t, err) 930 931 t.Log("Check that the directories match on the 2 users.") 932 933 children1, err := kbfsOps1.GetDirChildren(ctx, dirA1) 934 require.NoError(t, err) 935 936 children2, err := kbfsOps2.GetDirChildren(ctx, dirA2) 937 require.NoError(t, err) 938 939 require.Equal(t, children2, children1) 940 941 t.Log("Verify we can access the conflict folder through another handle") 942 dateStr := now.UTC().Format("2006-01-02") 943 h, err := tlfhandle.ParseHandle( 944 ctx, config2.KBPKI(), config2.MDOps(), nil, 945 name+" (local conflicted copy "+dateStr+")", tlf.Private) 946 require.NoError(t, err) 947 b, ok := data.MakeConflictBranchName(h) 948 require.True(t, ok) 949 950 rootNodeConflict, _, err := kbfsOps2.GetRootNode(ctx, h, b) 951 require.NoError(t, err) 952 dirAConflict, _, err := kbfsOps2.Lookup(ctx, rootNodeConflict, testPPS("a")) 953 require.NoError(t, err) 954 fileBConflict, _, err := kbfsOps2.Lookup(ctx, dirAConflict, testPPS("b")) 955 require.NoError(t, err) 956 957 gotData2 := make([]byte, len(data2)) 958 _, err = kbfsOps2.Read(ctx, fileBConflict, gotData2, 0) 959 require.NoError(t, err) 960 require.Equal(t, data2, gotData2) 961} 962 963// Tests that two users can create the same file simultaneously, and 964// the unmerged user can write to it, and they will be merged into a 965// single file. 966func TestBasicCRFileCreateUnmergedWriteConflict(t *testing.T) { 967 // simulate two users 968 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 969 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 970 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 971 972 config2 := ConfigAsUser(config1, userName2) 973 defer CheckConfigAndShutdown(ctx, t, config2) 974 975 config2.SetClock(clocktest.NewTestClockNow()) 976 977 name := userName1.String() + "," + userName2.String() 978 979 // user1 creates a file in a shared dir 980 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 981 982 kbfsOps1 := config1.KBFSOps() 983 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 984 require.NoError(t, err) 985 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 986 require.NoError(t, err) 987 988 // look it up on user2 989 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 990 991 kbfsOps2 := config2.KBFSOps() 992 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 993 require.NoError(t, err) 994 // disable updates and CR on user 2 995 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 996 require.NoError(t, err) 997 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 998 require.NoError(t, err) 999 1000 // User 1 creates a file 1001 _, _, err = kbfsOps1.CreateFile(ctx, dirA1, testPPS("b"), false, NoExcl) 1002 require.NoError(t, err) 1003 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 1004 require.NoError(t, err) 1005 1006 // User 2 creates the same file, and writes to it. 1007 fileB2, _, err := kbfsOps2.CreateFile( 1008 ctx, dirA2, testPPS("b"), false, NoExcl) 1009 require.NoError(t, err) 1010 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1011 require.NoError(t, err) 1012 data2 := []byte{5, 4, 3, 2, 1} 1013 err = kbfsOps2.Write(ctx, fileB2, data2, 0) 1014 require.NoError(t, err) 1015 err = kbfsOps2.SyncAll(ctx, fileB2.GetFolderBranch()) 1016 require.NoError(t, err) 1017 1018 // re-enable updates, and wait for CR to complete 1019 c <- struct{}{} 1020 err = RestartCRForTesting( 1021 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1022 rootNode2.GetFolderBranch()) 1023 require.NoError(t, err) 1024 err = kbfsOps2.SyncFromServer(ctx, 1025 rootNode2.GetFolderBranch(), nil) 1026 require.NoError(t, err) 1027 1028 err = kbfsOps1.SyncFromServer(ctx, 1029 rootNode1.GetFolderBranch(), nil) 1030 require.NoError(t, err) 1031 1032 // Make sure they both see the same set of children 1033 expectedChildren := []string{ 1034 "b", 1035 } 1036 children1, err := kbfsOps1.GetDirChildren(ctx, dirA1) 1037 require.NoError(t, err) 1038 1039 children2, err := kbfsOps2.GetDirChildren(ctx, dirA2) 1040 require.NoError(t, err) 1041 1042 assert.Equal(t, len(expectedChildren), len(children1)) 1043 1044 for _, child := range expectedChildren { 1045 _, ok := children1[dirA1.ChildName(child)] 1046 assert.True(t, ok) 1047 } 1048 1049 require.Equal(t, children1, children2) 1050} 1051 1052// Test that two conflict resolutions work correctly. 1053func TestCRDouble(t *testing.T) { 1054 // simulate two users 1055 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1056 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1057 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1058 config1.MDServer().DisableRekeyUpdatesForTesting() 1059 1060 config2 := ConfigAsUser(config1, userName2) 1061 defer CheckConfigAndShutdown(ctx, t, config2) 1062 _, err := config2.KBPKI().GetCurrentSession(context.Background()) 1063 require.NoError(t, err) 1064 config2.MDServer().DisableRekeyUpdatesForTesting() 1065 1066 config2.SetClock(clocktest.NewTestClockNow()) 1067 name := userName1.String() + "," + userName2.String() 1068 1069 // create and write to a file 1070 rootNode := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1071 kbfsOps1 := config1.KBFSOps() 1072 _, _, err = kbfsOps1.CreateFile(ctx, rootNode, testPPS("a"), false, NoExcl) 1073 require.NoError(t, err) 1074 err = kbfsOps1.SyncAll(ctx, rootNode.GetFolderBranch()) 1075 require.NoError(t, err) 1076 1077 // look it up on user2 1078 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1079 1080 kbfsOps2 := config2.KBFSOps() 1081 _, _, err = kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1082 require.NoError(t, err) 1083 // disable updates and CR on user 2 1084 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1085 require.NoError(t, err) 1086 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1087 require.NoError(t, err) 1088 1089 // User 1 creates a new file to start a conflict. 1090 _, _, err = kbfsOps1.CreateFile(ctx, rootNode, testPPS("b"), false, NoExcl) 1091 require.NoError(t, err) 1092 err = kbfsOps1.SyncAll(ctx, rootNode.GetFolderBranch()) 1093 require.NoError(t, err) 1094 1095 // User 2 makes a couple revisions 1096 fileNodeC, _, err := kbfsOps2.CreateFile( 1097 ctx, rootNode2, testPPS("c"), false, NoExcl) 1098 require.NoError(t, err) 1099 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1100 require.NoError(t, err) 1101 err = kbfsOps2.Write(ctx, fileNodeC, []byte{0}, 0) 1102 require.NoError(t, err) 1103 1104 var wg sync.WaitGroup 1105 syncCtx, cancel := context.WithCancel(ctx) 1106 1107 // Cancel this revision after the Put happens, to force the 1108 // background block manager to try to clean up. 1109 onSyncStalledCh, syncUnstallCh, syncCtx := StallMDOp( 1110 syncCtx, config2, StallableMDAfterPutUnmerged, 1) 1111 1112 wg.Add(1) 1113 go func() { 1114 defer wg.Done() 1115 err = kbfsOps2.SyncAll(syncCtx, fileNodeC.GetFolderBranch()) 1116 // Even though internally folderBranchOps ignores the 1117 // cancellation error when putting on an unmerged branch, the 1118 // wrapper function *might* still return it. 1119 if err != nil { 1120 assert.Equal(t, context.Canceled, err) 1121 } 1122 }() 1123 <-onSyncStalledCh 1124 cancel() 1125 close(syncUnstallCh) 1126 wg.Wait() 1127 1128 // Sync for real to clear out the dirty files. 1129 err = kbfsOps2.SyncAll(ctx, fileNodeC.GetFolderBranch()) 1130 require.NoError(t, err) 1131 1132 // Do one CR. 1133 c <- struct{}{} 1134 err = RestartCRForTesting( 1135 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1136 rootNode2.GetFolderBranch()) 1137 require.NoError(t, err) 1138 err = kbfsOps2.SyncFromServer(ctx, 1139 rootNode2.GetFolderBranch(), nil) 1140 require.NoError(t, err) 1141 1142 // A few merged revisions 1143 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("e"), false, NoExcl) 1144 require.NoError(t, err) 1145 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1146 require.NoError(t, err) 1147 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("f"), false, NoExcl) 1148 require.NoError(t, err) 1149 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1150 require.NoError(t, err) 1151 1152 ops := getOps(config2, rootNode.GetFolderBranch().Tlf) 1153 // Wait for the processor to try to delete the failed revision 1154 // (which pulls the unmerged MD ops back into the cache). 1155 err = ops.fbm.waitForArchives(ctx) 1156 require.NoError(t, err) 1157 err = ops.fbm.waitForDeletingBlocks(ctx) 1158 require.NoError(t, err) 1159 1160 // Sync user 1, then start another round of CR. 1161 err = kbfsOps1.SyncFromServer(ctx, 1162 rootNode2.GetFolderBranch(), nil) 1163 require.NoError(t, err) 1164 // disable updates and CR on user 2 1165 c, err = DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1166 require.NoError(t, err) 1167 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1168 require.NoError(t, err) 1169 _, _, err = kbfsOps1.CreateFile(ctx, rootNode, testPPS("g"), false, NoExcl) 1170 require.NoError(t, err) 1171 err = kbfsOps1.SyncAll(ctx, rootNode.GetFolderBranch()) 1172 require.NoError(t, err) 1173 1174 // User 2 makes a couple unmerged revisions 1175 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("h"), false, NoExcl) 1176 require.NoError(t, err) 1177 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1178 require.NoError(t, err) 1179 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("i"), false, NoExcl) 1180 require.NoError(t, err) 1181 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1182 require.NoError(t, err) 1183 1184 // Do a second CR. 1185 c <- struct{}{} 1186 err = RestartCRForTesting( 1187 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1188 rootNode2.GetFolderBranch()) 1189 require.NoError(t, err) 1190 err = kbfsOps2.SyncFromServer(ctx, 1191 rootNode2.GetFolderBranch(), nil) 1192 require.NoError(t, err) 1193} 1194 1195// Tests that two users can make independent writes while forked, and 1196// conflict resolution will merge them correctly and the rekey bit is 1197// preserved until rekey. 1198func TestBasicCRFileConflictWithRekey(t *testing.T) { 1199 // simulate two users 1200 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1201 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1202 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1203 config1.MDServer().DisableRekeyUpdatesForTesting() 1204 1205 config2 := ConfigAsUser(config1, userName2) 1206 defer CheckConfigAndShutdown(ctx, t, config2) 1207 session2, err := config2.KBPKI().GetCurrentSession(context.Background()) 1208 require.NoError(t, err) 1209 config2.MDServer().DisableRekeyUpdatesForTesting() 1210 1211 clock, now := clocktest.NewTestClockAndTimeNow() 1212 config2.SetClock(clock) 1213 name := userName1.String() + "," + userName2.String() 1214 1215 // user1 creates a file in a shared dir 1216 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1217 1218 kbfsOps1 := config1.KBFSOps() 1219 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 1220 require.NoError(t, err) 1221 fileB1, _, err := kbfsOps1.CreateFile( 1222 ctx, dirA1, testPPS("b"), false, NoExcl) 1223 require.NoError(t, err) 1224 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 1225 require.NoError(t, err) 1226 1227 // look it up on user2 1228 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1229 1230 kbfsOps2 := config2.KBFSOps() 1231 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1232 require.NoError(t, err) 1233 fileB2, _, err := kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 1234 require.NoError(t, err) 1235 1236 config2Dev2 := ConfigAsUser(config1, userName2) 1237 // we don't check the config because this device can't read all of 1238 // the md blocks. 1239 defer func() { _ = config2Dev2.Shutdown(ctx) }() 1240 config2Dev2.MDServer().DisableRekeyUpdatesForTesting() 1241 1242 // Now give u2 a new device. The configs don't share a Keybase 1243 // Daemon so we have to do it in all places. 1244 AddDeviceForLocalUserOrBust(t, config1, session2.UID) 1245 AddDeviceForLocalUserOrBust(t, config2, session2.UID) 1246 devIndex := AddDeviceForLocalUserOrBust(t, config2Dev2, session2.UID) 1247 SwitchDeviceForLocalUserOrBust(t, config2Dev2, devIndex) 1248 1249 // user2 device 2 should be unable to read the data now since its device 1250 // wasn't registered when the folder was originally created. 1251 _, err = GetRootNodeForTest(ctx, config2Dev2, name, tlf.Private) 1252 require.IsType(t, NeedSelfRekeyError{}, err) 1253 1254 // User 2 syncs 1255 err = kbfsOps2.SyncFromServer(ctx, 1256 rootNode2.GetFolderBranch(), nil) 1257 require.NoError(t, err) 1258 1259 // disable updates on user2 1260 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1261 require.NoError(t, err) 1262 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1263 require.NoError(t, err) 1264 1265 // User 1 writes the file 1266 data1 := []byte{1, 2, 3, 4, 5} 1267 err = kbfsOps1.Write(ctx, fileB1, data1, 0) 1268 require.NoError(t, err) 1269 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 1270 require.NoError(t, err) 1271 1272 // User 2 dev 2 should set the rekey bit 1273 kbfsOps2Dev2 := config2Dev2.KBFSOps() 1274 _, _ = RequestRekeyAndWaitForOneFinishEvent(ctx, 1275 kbfsOps2Dev2, rootNode2.GetFolderBranch().Tlf) 1276 1277 // User 1 syncs 1278 err = kbfsOps1.SyncFromServer(ctx, 1279 rootNode1.GetFolderBranch(), nil) 1280 require.NoError(t, err) 1281 1282 // User 2 makes a new different file 1283 data2 := []byte{5, 4, 3, 2, 1} 1284 err = kbfsOps2.Write(ctx, fileB2, data2, 0) 1285 require.NoError(t, err) 1286 err = kbfsOps2.SyncAll(ctx, fileB2.GetFolderBranch()) 1287 require.NoError(t, err) 1288 1289 // re-enable updates, and wait for CR to complete. 1290 // this should also cause a rekey of the folder. 1291 c <- struct{}{} 1292 err = RestartCRForTesting( 1293 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1294 rootNode2.GetFolderBranch()) 1295 require.NoError(t, err) 1296 err = kbfsOps2.SyncFromServer(ctx, 1297 rootNode2.GetFolderBranch(), nil) 1298 require.NoError(t, err) 1299 // wait for the rekey to happen 1300 _, _ = RequestRekeyAndWaitForOneFinishEvent(ctx, 1301 config2.KBFSOps(), rootNode2.GetFolderBranch().Tlf) 1302 1303 err = kbfsOps1.SyncFromServer(ctx, 1304 rootNode1.GetFolderBranch(), nil) 1305 require.NoError(t, err) 1306 1307 // Look it up on user 2 dev 2 after syncing. 1308 err = kbfsOps2Dev2.SyncFromServer(ctx, 1309 rootNode2.GetFolderBranch(), nil) 1310 require.NoError(t, err) 1311 rootNode2Dev2 := GetRootNodeOrBust(ctx, t, config2Dev2, name, tlf.Private) 1312 dirA2Dev2, _, err := kbfsOps2Dev2.Lookup(ctx, rootNode2Dev2, testPPS("a")) 1313 require.NoError(t, err) 1314 1315 cre := WriterDeviceDateConflictRenamer{} 1316 // Make sure they all see the same set of children 1317 expectedChildren := []string{ 1318 "b", 1319 cre.ConflictRenameHelper(now, "u2", "dev1", "b"), 1320 } 1321 children1, err := kbfsOps1.GetDirChildren(ctx, dirA1) 1322 require.NoError(t, err) 1323 1324 children2, err := kbfsOps2.GetDirChildren(ctx, dirA2) 1325 require.NoError(t, err) 1326 1327 children2Dev2, err := kbfsOps2Dev2.GetDirChildren(ctx, dirA2Dev2) 1328 require.NoError(t, err) 1329 1330 assert.Equal(t, len(expectedChildren), len(children1)) 1331 1332 for _, child := range expectedChildren { 1333 _, ok := children1[dirA1.ChildName(child)] 1334 assert.True(t, ok) 1335 } 1336 1337 require.Equal(t, children1, children2) 1338 require.Equal(t, children2, children2Dev2) 1339} 1340 1341// Same as above, except the "winner" is the rekey request, and the 1342// "loser" is the file write. Regression test for KBFS-773. 1343func TestBasicCRFileConflictWithMergedRekey(t *testing.T) { 1344 // simulate two users 1345 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1346 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1347 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1348 config1.MDServer().DisableRekeyUpdatesForTesting() 1349 1350 config2 := ConfigAsUser(config1, userName2) 1351 defer CheckConfigAndShutdown(ctx, t, config2) 1352 session2, err := config2.KBPKI().GetCurrentSession(context.Background()) 1353 require.NoError(t, err) 1354 config2.MDServer().DisableRekeyUpdatesForTesting() 1355 1356 config2.SetClock(clocktest.NewTestClockNow()) 1357 name := userName1.String() + "," + userName2.String() 1358 1359 // user1 creates a file in a shared dir 1360 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1361 1362 kbfsOps1 := config1.KBFSOps() 1363 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 1364 require.NoError(t, err) 1365 fileB1, _, err := kbfsOps1.CreateFile( 1366 ctx, dirA1, testPPS("b"), false, NoExcl) 1367 require.NoError(t, err) 1368 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 1369 require.NoError(t, err) 1370 1371 // look it up on user2 1372 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1373 1374 kbfsOps2 := config2.KBFSOps() 1375 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1376 require.NoError(t, err) 1377 1378 config2Dev2 := ConfigAsUser(config1, userName2) 1379 // we don't check the config because this device can't read all of 1380 // the md blocks. 1381 defer func() { _ = config2Dev2.Shutdown(ctx) }() 1382 config2Dev2.MDServer().DisableRekeyUpdatesForTesting() 1383 1384 // Now give u2 a new device. The configs don't share a Keybase 1385 // Daemon so we have to do it in all places. 1386 AddDeviceForLocalUserOrBust(t, config1, session2.UID) 1387 AddDeviceForLocalUserOrBust(t, config2, session2.UID) 1388 devIndex := AddDeviceForLocalUserOrBust(t, config2Dev2, session2.UID) 1389 SwitchDeviceForLocalUserOrBust(t, config2Dev2, devIndex) 1390 1391 // user2 device 2 should be unable to read the data now since its device 1392 // wasn't registered when the folder was originally created. 1393 _, err = GetRootNodeForTest(ctx, config2Dev2, name, tlf.Private) 1394 require.IsType(t, NeedSelfRekeyError{}, err) 1395 1396 // User 2 syncs 1397 err = kbfsOps2.SyncFromServer(ctx, 1398 rootNode2.GetFolderBranch(), nil) 1399 require.NoError(t, err) 1400 1401 // disable updates on user1 1402 c, err := DisableUpdatesForTesting(config1, rootNode2.GetFolderBranch()) 1403 require.NoError(t, err) 1404 err = DisableCRForTesting(config1, rootNode2.GetFolderBranch()) 1405 require.NoError(t, err) 1406 1407 // User 2 dev 2 should set the rekey bit 1408 kbfsOps2Dev2 := config2Dev2.KBFSOps() 1409 _, _ = RequestRekeyAndWaitForOneFinishEvent(ctx, 1410 kbfsOps2Dev2, rootNode2.GetFolderBranch().Tlf) 1411 1412 // User 1 writes the file 1413 data1 := []byte{1, 2, 3, 4, 5} 1414 err = kbfsOps1.Write(ctx, fileB1, data1, 0) 1415 require.NoError(t, err) 1416 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 1417 require.NoError(t, err) 1418 1419 // re-enable updates, and wait for CR to complete. 1420 // this should also cause a rekey of the folder. 1421 c <- struct{}{} 1422 err = RestartCRForTesting( 1423 libcontext.BackgroundContextWithCancellationDelayer(), config1, 1424 rootNode2.GetFolderBranch()) 1425 require.NoError(t, err) 1426 err = kbfsOps1.SyncFromServer(ctx, 1427 rootNode1.GetFolderBranch(), nil) 1428 require.NoError(t, err) 1429 // wait for the rekey to happen 1430 _, _ = RequestRekeyAndWaitForOneFinishEvent(ctx, 1431 config1.KBFSOps(), rootNode1.GetFolderBranch().Tlf) 1432 1433 err = kbfsOps1.SyncFromServer(ctx, 1434 rootNode1.GetFolderBranch(), nil) 1435 require.NoError(t, err) 1436 1437 err = kbfsOps2.SyncFromServer(ctx, 1438 rootNode2.GetFolderBranch(), nil) 1439 require.NoError(t, err) 1440 1441 // Look it up on user 2 dev 2 after syncing. 1442 err = kbfsOps2Dev2.SyncFromServer(ctx, 1443 rootNode2.GetFolderBranch(), nil) 1444 require.NoError(t, err) 1445 rootNode2Dev2 := GetRootNodeOrBust(ctx, t, config2Dev2, name, tlf.Private) 1446 dirA2Dev2, _, err := kbfsOps2Dev2.Lookup(ctx, rootNode2Dev2, testPPS("a")) 1447 require.NoError(t, err) 1448 1449 // Make sure they all see the same set of children 1450 expectedChildren := []string{ 1451 "b", 1452 } 1453 children1, err := kbfsOps1.GetDirChildren(ctx, dirA1) 1454 require.NoError(t, err) 1455 1456 children2, err := kbfsOps2.GetDirChildren(ctx, dirA2) 1457 require.NoError(t, err) 1458 1459 children2Dev2, err := kbfsOps2Dev2.GetDirChildren(ctx, dirA2Dev2) 1460 require.NoError(t, err) 1461 1462 assert.Equal(t, len(expectedChildren), len(children1)) 1463 1464 for _, child := range expectedChildren { 1465 _, ok := children1[dirA1.ChildName(child)] 1466 assert.True(t, ok) 1467 } 1468 1469 require.Equal(t, children1, children2) 1470 require.Equal(t, children2, children2Dev2) 1471} 1472 1473// Test that, when writing multiple blocks in parallel under conflict 1474// resolution, one error will cancel the remaining puts and the block 1475// server will be consistent. 1476func TestCRSyncParallelBlocksErrorCleanup(t *testing.T) { 1477 t.Skip("Broken due to KBFS-1193") 1478 1479 // simulate two users 1480 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1481 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1482 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1483 config1.MDServer().DisableRekeyUpdatesForTesting() 1484 1485 config2 := ConfigAsUser(config1, userName2) 1486 defer CheckConfigAndShutdown(ctx, t, config2) 1487 _, err := config2.KBPKI().GetCurrentSession(context.Background()) 1488 require.NoError(t, err) 1489 config2.MDServer().DisableRekeyUpdatesForTesting() 1490 1491 config2.SetClock(clocktest.NewTestClockNow()) 1492 name := userName1.String() + "," + userName2.String() 1493 1494 // Make the blocks small, with multiple levels of indirection, but 1495 // make the unembedded size large, so we don't create thousands of 1496 // unembedded block change blocks. 1497 blockSize := int64(5) 1498 bsplit, err := data.NewBlockSplitterSimpleExact(blockSize, 2, 100*1024) 1499 require.NoError(t, err) 1500 config1.SetBlockSplitter(bsplit) 1501 1502 // create and write to a file 1503 rootNode := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1504 kbfsOps1 := config1.KBFSOps() 1505 _, _, err = kbfsOps1.CreateFile(ctx, rootNode, testPPS("a"), false, NoExcl) 1506 require.NoError(t, err) 1507 err = kbfsOps1.SyncAll(ctx, rootNode.GetFolderBranch()) 1508 require.NoError(t, err) 1509 1510 // look it up on user2 1511 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1512 1513 kbfsOps2 := config2.KBFSOps() 1514 _, _, err = kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1515 require.NoError(t, err) 1516 // disable updates and CR on user 2 1517 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1518 require.NoError(t, err) 1519 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1520 require.NoError(t, err) 1521 1522 // User 1 creates a new file to start a conflict. 1523 _, _, err = kbfsOps1.CreateFile(ctx, rootNode, testPPS("b"), false, NoExcl) 1524 require.NoError(t, err) 1525 err = kbfsOps1.SyncAll(ctx, rootNode.GetFolderBranch()) 1526 require.NoError(t, err) 1527 1528 // User 2 does one successful operation to create the first unmerged MD. 1529 fileNodeB, _, err := kbfsOps2.CreateFile( 1530 ctx, rootNode2, testPPS("b"), false, NoExcl) 1531 require.NoError(t, err) 1532 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1533 require.NoError(t, err) 1534 1535 // User 2 writes some data 1536 fileBlocks := int64(maxParallelBlockPuts + 5) 1537 var data []byte 1538 for i := int64(0); i < blockSize*fileBlocks; i++ { 1539 data = append(data, byte(i)) 1540 } 1541 err = kbfsOps2.Write(ctx, fileNodeB, data, 0) 1542 require.NoError(t, err) 1543 1544 // Start the sync and wait for it to stall. 1545 var wg sync.WaitGroup 1546 wg.Add(1) 1547 syncCtx, cancel := context.WithCancel( 1548 libcontext.BackgroundContextWithCancellationDelayer()) 1549 defer func() { 1550 err := libcontext.CleanupCancellationDelayer(syncCtx) 1551 require.NoError(t, err) 1552 }() 1553 1554 // Now user 2 makes a big write where most of the blocks get canceled. 1555 // We only need to know the first time we stall. 1556 onSyncStalledCh, syncUnstallCh, syncCtx := StallBlockOp( 1557 syncCtx, config2, StallableBlockPut, 2) 1558 1559 var syncErr error 1560 go func() { 1561 defer wg.Done() 1562 1563 syncErr = kbfsOps2.SyncAll(syncCtx, fileNodeB.GetFolderBranch()) 1564 }() 1565 // Wait for 2 of the blocks and let them go 1566 <-onSyncStalledCh 1567 <-onSyncStalledCh 1568 syncUnstallCh <- struct{}{} 1569 syncUnstallCh <- struct{}{} 1570 1571 // Wait for the rest of the puts (this indicates that the first 1572 // two succeeded correctly and two more were sent to replace them) 1573 for i := 0; i < maxParallelBlockPuts; i++ { 1574 <-onSyncStalledCh 1575 } 1576 // Cancel so all other block puts fail 1577 cancel() 1578 close(syncUnstallCh) 1579 wg.Wait() 1580 1581 require.Equal(t, context.Canceled, syncErr) 1582 1583 // Get the mdWriterLock to be sure the sync has exited (since the 1584 // cleanup logic happens in a background goroutine) 1585 ops := getOps(config2, rootNode2.GetFolderBranch().Tlf) 1586 lState := makeFBOLockState() 1587 ops.mdWriterLock.Lock(lState) 1588 ops.mdWriterLock.Unlock(lState) 1589 1590 // The state checker will make sure those blocks from 1591 // the failed sync get cleaned up. 1592 1593 for i := int64(0); i < blockSize*fileBlocks; i++ { 1594 data[i] = byte(i + 10) 1595 } 1596 err = kbfsOps2.Write(ctx, fileNodeB, data, 0) 1597 require.NoError(t, err) 1598 err = kbfsOps2.SyncAll(ctx, fileNodeB.GetFolderBranch()) 1599 require.NoError(t, err) 1600 1601 c <- struct{}{} 1602 err = RestartCRForTesting( 1603 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1604 rootNode2.GetFolderBranch()) 1605 require.NoError(t, err) 1606 err = kbfsOps2.SyncFromServer(ctx, 1607 rootNode2.GetFolderBranch(), nil) 1608 require.NoError(t, err) 1609} 1610 1611// Test that a resolution can be canceled right before the Put due to 1612// another operation, and then the second resolution includes both 1613// unmerged operations. Regression test for KBFS-1133. 1614func TestCRCanceledAfterNewOperation(t *testing.T) { 1615 // simulate two users 1616 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1617 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1618 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1619 config1.MDServer().DisableRekeyUpdatesForTesting() 1620 1621 config2 := ConfigAsUser(config1, userName2) 1622 defer CheckConfigAndShutdown(ctx, t, config2) 1623 _, err := config2.KBPKI().GetCurrentSession(context.Background()) 1624 require.NoError(t, err) 1625 config2.MDServer().DisableRekeyUpdatesForTesting() 1626 1627 clock, now := clocktest.NewTestClockAndTimeNow() 1628 config2.SetClock(clock) 1629 name := userName1.String() + "," + userName2.String() 1630 1631 // create and write to a file 1632 rootNode := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1633 kbfsOps1 := config1.KBFSOps() 1634 aNode1, _, err := kbfsOps1.CreateFile( 1635 ctx, rootNode, testPPS("a"), false, NoExcl) 1636 require.NoError(t, err) 1637 data := []byte{1, 2, 3, 4, 5} 1638 err = kbfsOps1.Write(ctx, aNode1, data, 0) 1639 require.NoError(t, err) 1640 err = kbfsOps1.SyncAll(ctx, aNode1.GetFolderBranch()) 1641 require.NoError(t, err) 1642 1643 // look it up on user2 1644 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1645 1646 kbfsOps2 := config2.KBFSOps() 1647 aNode2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1648 require.NoError(t, err) 1649 // disable updates and CR on user 2 1650 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1651 require.NoError(t, err) 1652 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1653 require.NoError(t, err) 1654 1655 // User 1 truncates file a. 1656 err = kbfsOps1.Truncate(ctx, aNode1, 0) 1657 require.NoError(t, err) 1658 err = kbfsOps1.SyncAll(ctx, aNode1.GetFolderBranch()) 1659 require.NoError(t, err) 1660 1661 // User 2 writes to the file, creating a conflict. 1662 data2 := []byte{5, 4, 3, 2, 1} 1663 err = kbfsOps2.Write(ctx, aNode2, data2, 0) 1664 require.NoError(t, err) 1665 err = kbfsOps2.SyncAll(ctx, aNode2.GetFolderBranch()) 1666 require.NoError(t, err) 1667 1668 onPutStalledCh, putUnstallCh, putCtx := 1669 StallMDOp(context.Background(), config2, StallableMDResolveBranch, 1) 1670 1671 var wg sync.WaitGroup 1672 putCtx, cancel2 := context.WithCancel(putCtx) 1673 wg.Add(1) 1674 go func() { 1675 defer wg.Done() 1676 1677 c <- struct{}{} 1678 // Make sure the CR gets done with a context we can use for 1679 // stalling. 1680 err = RestartCRForTesting(putCtx, config2, 1681 rootNode2.GetFolderBranch()) 1682 assert.NoError(t, err) 1683 err = kbfsOps2.SyncFromServer(putCtx, 1684 rootNode2.GetFolderBranch(), nil) 1685 assert.Error(t, err) 1686 }() 1687 <-onPutStalledCh 1688 cancel2() 1689 close(putUnstallCh) 1690 wg.Wait() 1691 1692 // Disable again 1693 c, err = DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1694 require.NoError(t, err) 1695 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1696 require.NoError(t, err) 1697 1698 // Do a second operation and complete the resolution. 1699 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("b"), false, NoExcl) 1700 require.NoError(t, err) 1701 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1702 require.NoError(t, err) 1703 c <- struct{}{} 1704 err = RestartCRForTesting( 1705 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1706 rootNode2.GetFolderBranch()) 1707 require.NoError(t, err) 1708 err = kbfsOps2.SyncFromServer(ctx, 1709 rootNode2.GetFolderBranch(), nil) 1710 require.NoError(t, err) 1711 1712 // Now there should be a conflict file containing data2. 1713 cre := WriterDeviceDateConflictRenamer{} 1714 // Make sure they both see the same set of children 1715 expectedChildren := []string{ 1716 "a", 1717 cre.ConflictRenameHelper(now, "u2", "dev1", "a"), 1718 "b", 1719 } 1720 children2, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 1721 require.NoError(t, err) 1722 assert.Equal(t, len(expectedChildren), len(children2)) 1723 for _, child := range expectedChildren { 1724 _, ok := children2[rootNode2.ChildName(child)] 1725 assert.True(t, ok) 1726 } 1727} 1728 1729// Tests that if a user gets /too/ unmerged, they will have their 1730// unmerged writes blocked. 1731func TestBasicCRBlockUnmergedWrites(t *testing.T) { 1732 // simulate two users 1733 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1734 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1735 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1736 1737 config2 := ConfigAsUser(config1, userName2) 1738 defer CheckConfigAndShutdown(ctx, t, config2) 1739 1740 name := userName1.String() + "," + userName2.String() 1741 1742 // user1 creates a file in a shared dir 1743 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1744 1745 kbfsOps1 := config1.KBFSOps() 1746 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 1747 require.NoError(t, err) 1748 _, _, err = kbfsOps1.CreateFile(ctx, dirA1, testPPS("b"), false, NoExcl) 1749 require.NoError(t, err) 1750 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 1751 require.NoError(t, err) 1752 1753 // look it up on user2 1754 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1755 1756 kbfsOps2 := config2.KBFSOps() 1757 ops2 := getOps(config2, rootNode2.GetFolderBranch().Tlf) 1758 ops2.cr.maxRevsThreshold = 2 1759 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1760 require.NoError(t, err) 1761 _, _, err = kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 1762 require.NoError(t, err) 1763 1764 // disable updates on user 2 1765 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1766 require.NoError(t, err) 1767 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1768 require.NoError(t, err) 1769 1770 // One write for user 1 1771 _, _, err = kbfsOps1.CreateFile(ctx, dirA1, testPPS("c"), false, NoExcl) 1772 require.NoError(t, err) 1773 err = kbfsOps1.SyncAll(ctx, dirA1.GetFolderBranch()) 1774 require.NoError(t, err) 1775 1776 // Two writes for user 2 1777 _, _, err = kbfsOps2.CreateFile(ctx, dirA2, testPPS("d"), false, NoExcl) 1778 require.NoError(t, err) 1779 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1780 require.NoError(t, err) 1781 _, _, err = kbfsOps2.CreateFile(ctx, dirA2, testPPS("e"), false, NoExcl) 1782 require.NoError(t, err) 1783 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1784 require.NoError(t, err) 1785 1786 // Start CR, but cancel it before it completes, which should lead 1787 // to it locking next time (since it has seen how many revisions 1788 // are outstanding). 1789 onPutStalledCh, putUnstallCh, putCtx := 1790 StallMDOp(context.Background(), config2, StallableMDResolveBranch, 1) 1791 1792 var wg sync.WaitGroup 1793 firstPutCtx, cancel := context.WithCancel(putCtx) 1794 wg.Add(1) 1795 go func() { 1796 defer wg.Done() 1797 1798 // Make sure the CR gets done with a context we can use for 1799 // stalling. 1800 err = RestartCRForTesting(firstPutCtx, config2, 1801 rootNode2.GetFolderBranch()) 1802 if !assert.NoError(t, err) { 1803 return 1804 } 1805 err = kbfsOps2.SyncFromServer(firstPutCtx, 1806 rootNode2.GetFolderBranch(), nil) 1807 if !assert.Error(t, err) { 1808 return 1809 } 1810 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1811 if !assert.NoError(t, err) { 1812 return 1813 } 1814 }() 1815 <-onPutStalledCh 1816 cancel() 1817 putUnstallCh <- struct{}{} 1818 wg.Wait() 1819 1820 // Pretend that CR was canceled by another write. 1821 _, _, err = kbfsOps2.CreateFile(ctx, dirA2, testPPS("f"), false, NoExcl) 1822 require.NoError(t, err) 1823 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1824 require.NoError(t, err) 1825 1826 // Now restart CR, and make sure it blocks all writes. 1827 wg.Add(1) 1828 go func() { 1829 defer wg.Done() 1830 1831 // Make sure the CR gets done with a context we can use for 1832 // stalling. 1833 err = RestartCRForTesting(putCtx, config2, 1834 rootNode2.GetFolderBranch()) 1835 if !assert.NoError(t, err) { 1836 return 1837 } 1838 }() 1839 <-onPutStalledCh 1840 c <- struct{}{} 1841 1842 // Now try to write again 1843 writeErrCh := make(chan error, 1) 1844 go func() { 1845 _, _, err := kbfsOps2.CreateFile( 1846 ctx, dirA2, testPPS("g"), false, NoExcl) 1847 assert.NoError(t, err) 1848 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1849 require.NoError(t, err) 1850 writeErrCh <- err 1851 }() 1852 1853 // For now, assume we're blocked if the write doesn't go through 1854 // in 20ms. TODO: instruct mdWriterLock to know for sure how many 1855 // goroutines it's blocking? 1856 timer := time.After(20 * time.Millisecond) 1857 select { 1858 case <-writeErrCh: 1859 t.Fatalf("Write finished without blocking") 1860 case <-timer: 1861 } 1862 1863 // Finish the CR. 1864 close(putUnstallCh) 1865 wg.Wait() 1866 1867 // Now the write can finish 1868 err = <-writeErrCh 1869 require.NoError(t, err) 1870} 1871 1872// Test that an umerged put can be canceled, and the conflict 1873// resolution will fix the resulting weird state. 1874func TestUnmergedPutAfterCanceledUnmergedPut(t *testing.T) { 1875 // simulate two users 1876 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 1877 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 1878 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 1879 config1.MDServer().DisableRekeyUpdatesForTesting() 1880 1881 config2 := ConfigAsUser(config1, userName2) 1882 defer CheckConfigAndShutdown(ctx, t, config2) 1883 _, err := config2.KBPKI().GetCurrentSession(context.Background()) 1884 require.NoError(t, err) 1885 config2.MDServer().DisableRekeyUpdatesForTesting() 1886 1887 name := userName1.String() + "," + userName2.String() 1888 1889 // create and write to a file 1890 rootNode := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 1891 kbfsOps1 := config1.KBFSOps() 1892 aNode1, _, err := kbfsOps1.CreateFile( 1893 ctx, rootNode, testPPS("a"), false, NoExcl) 1894 require.NoError(t, err) 1895 data := []byte{1, 2, 3, 4, 5} 1896 err = kbfsOps1.Write(ctx, aNode1, data, 0) 1897 require.NoError(t, err) 1898 err = kbfsOps1.SyncAll(ctx, aNode1.GetFolderBranch()) 1899 require.NoError(t, err) 1900 1901 // look it up on user2 1902 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 1903 1904 kbfsOps2 := config2.KBFSOps() 1905 _, _, err = kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 1906 require.NoError(t, err) 1907 // disable updates and CR on user 2 1908 c, err := DisableUpdatesForTesting(config2, rootNode2.GetFolderBranch()) 1909 require.NoError(t, err) 1910 err = DisableCRForTesting(config2, rootNode2.GetFolderBranch()) 1911 require.NoError(t, err) 1912 1913 // User 1 truncates file a. 1914 err = kbfsOps1.Truncate(ctx, aNode1, 0) 1915 require.NoError(t, err) 1916 err = kbfsOps1.SyncAll(ctx, aNode1.GetFolderBranch()) 1917 require.NoError(t, err) 1918 1919 // User 2 creates a file to start a conflict branch. 1920 _, _, err = kbfsOps2.CreateFile(ctx, rootNode2, testPPS("b"), false, NoExcl) 1921 require.NoError(t, err) 1922 err = kbfsOps2.SyncAll(ctx, rootNode2.GetFolderBranch()) 1923 require.NoError(t, err) 1924 1925 onPutStalledCh, putUnstallCh, putCtx := 1926 StallMDOp(ctx, config2, StallableMDPutUnmerged, 1) 1927 1928 var wg sync.WaitGroup 1929 putCtx, cancel2 := context.WithCancel(putCtx) 1930 wg.Add(1) 1931 go func() { 1932 defer wg.Done() 1933 _, _, err = kbfsOps2.CreateFile( 1934 putCtx, rootNode2, testPPS("c"), false, NoExcl) 1935 require.NoError(t, err) 1936 err = kbfsOps2.SyncAll(putCtx, rootNode2.GetFolderBranch()) 1937 // Even though internally folderBranchOps ignores the 1938 // cancellation error when putting on an unmerged branch, the 1939 // wrapper function *might* still return it. 1940 if err != nil { 1941 assert.Equal(t, context.Canceled, err) 1942 } 1943 1944 }() 1945 <-onPutStalledCh 1946 cancel2() 1947 close(putUnstallCh) 1948 wg.Wait() 1949 1950 // At this point, the local unmerged head doesn't match the 1951 // server's unmerged head, but CR will fix it up. 1952 1953 c <- struct{}{} 1954 err = RestartCRForTesting( 1955 libcontext.BackgroundContextWithCancellationDelayer(), config2, 1956 rootNode2.GetFolderBranch()) 1957 require.NoError(t, err) 1958 err = kbfsOps2.SyncFromServer(ctx, 1959 rootNode2.GetFolderBranch(), nil) 1960 require.NoError(t, err) 1961 1962 // Make sure they both see the same set of children. 1963 expectedChildren := []string{ 1964 "a", 1965 "b", 1966 "c", 1967 } 1968 children2, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 1969 require.NoError(t, err) 1970 assert.Equal(t, len(expectedChildren), len(children2)) 1971 for _, child := range expectedChildren { 1972 _, ok := children2[rootNode2.ChildName(child)] 1973 assert.True(t, ok) 1974 } 1975} 1976 1977func TestForceStuckConflict(t *testing.T) { 1978 tempdir, err := ioutil.TempDir(os.TempDir(), "journal_for_stuck_cr") 1979 defer os.RemoveAll(tempdir) 1980 require.NoError(t, err) 1981 1982 var u1 kbname.NormalizedUsername = "u1" 1983 config, _, ctx, cancel := kbfsOpsConcurInit(t, u1) 1984 defer kbfsConcurTestShutdown(ctx, t, config, cancel) 1985 1986 t.Log("Enable journaling") 1987 err = config.EnableDiskLimiter(tempdir) 1988 require.NoError(t, err) 1989 err = config.EnableJournaling( 1990 ctx, tempdir, TLFJournalBackgroundWorkEnabled) 1991 require.NoError(t, err) 1992 jManager, err := GetJournalManager(config) 1993 require.NoError(t, err) 1994 err = jManager.EnableAuto(ctx) 1995 require.NoError(t, err) 1996 1997 name := "u1" 1998 h, err := tlfhandle.ParseHandle( 1999 ctx, config.KBPKI(), config.MDOps(), nil, name, tlf.Private) 2000 require.NoError(t, err) 2001 kbfsOps := config.KBFSOps() 2002 2003 t.Log("Initialize the TLF") 2004 rootNode, _, err := kbfsOps.GetOrCreateRootNode(ctx, h, data.MasterBranch) 2005 require.NoError(t, err) 2006 _, _, err = kbfsOps.CreateDir(ctx, rootNode, testPPS("a")) 2007 require.NoError(t, err) 2008 err = kbfsOps.SyncAll(ctx, rootNode.GetFolderBranch()) 2009 require.NoError(t, err) 2010 2011 t.Log("Force a conflict") 2012 tlfID := rootNode.GetFolderBranch().Tlf 2013 err = kbfsOps.ForceStuckConflictForTesting(ctx, tlfID) 2014 require.NoError(t, err) 2015 2016 t.Log("Ensure conflict files are there") 2017 children, err := kbfsOps.GetDirChildren(ctx, rootNode) 2018 require.NoError(t, err) 2019 require.Len(t, children, 1+(maxConflictResolutionAttempts+1)) 2020 2021 t.Log("Ensure uploads can't be canceled") 2022 err = kbfsOps.CancelUploads(ctx, rootNode.GetFolderBranch()) 2023 require.Error(t, err) 2024 2025 t.Log("Clear conflict view") 2026 err = kbfsOps.ClearConflictView(ctx, tlfID) 2027 require.NoError(t, err) 2028 2029 t.Log("Ensure conflict files are gone") 2030 children, err = kbfsOps.GetDirChildren(ctx, rootNode) 2031 require.NoError(t, err) 2032 require.Len(t, children, 1) 2033} 2034 2035// Tests that if clearing a CR conflict can fast-forward if needed. 2036func TestBasicCRFailureClearAndFastForward(t *testing.T) { 2037 tempdir, err := ioutil.TempDir(os.TempDir(), "journal_for_fail_fix") 2038 defer os.RemoveAll(tempdir) 2039 2040 // simulate two users 2041 var userName1, userName2 kbname.NormalizedUsername = "u1", "u2" 2042 config1, _, ctx, cancel := kbfsOpsConcurInit(t, userName1, userName2) 2043 defer kbfsConcurTestShutdown(ctx, t, config1, cancel) 2044 2045 config2 := ConfigAsUser(config1, userName2) 2046 defer CheckConfigAndShutdown(ctx, t, config2) 2047 2048 // Enable journaling on user 2 2049 require.NoError(t, err) 2050 err = config2.EnableDiskLimiter(tempdir) 2051 require.NoError(t, err) 2052 err = config2.EnableJournaling(ctx, tempdir, 2053 TLFJournalBackgroundWorkEnabled) 2054 require.NoError(t, err) 2055 jManager, err := GetJournalManager(config2) 2056 require.NoError(t, err) 2057 err = jManager.EnableAuto(ctx) 2058 require.NoError(t, err) 2059 2060 name := userName1.String() + "," + userName2.String() 2061 2062 t.Log("User 1 creates a file a/b.") 2063 rootNode1 := GetRootNodeOrBust(ctx, t, config1, name, tlf.Private) 2064 2065 kbfsOps1 := config1.KBFSOps() 2066 dirA1, _, err := kbfsOps1.CreateDir(ctx, rootNode1, testPPS("a")) 2067 require.NoError(t, err) 2068 fileB1, _, err := kbfsOps1.CreateFile( 2069 ctx, dirA1, testPPS("b"), false, NoExcl) 2070 require.NoError(t, err) 2071 err = kbfsOps1.SyncAll(ctx, rootNode1.GetFolderBranch()) 2072 require.NoError(t, err) 2073 2074 t.Log("User 2 looks up the file node.") 2075 rootNode2 := GetRootNodeOrBust(ctx, t, config2, name, tlf.Private) 2076 kbfsOps2 := config2.KBFSOps() 2077 dirA2, _, err := kbfsOps2.Lookup(ctx, rootNode2, testPPS("a")) 2078 require.NoError(t, err) 2079 fileB2, _, err := kbfsOps2.Lookup(ctx, dirA2, testPPS("b")) 2080 require.NoError(t, err) 2081 2082 t.Log("Force a conflict") 2083 tlfID := rootNode2.GetFolderBranch().Tlf 2084 err = kbfsOps2.ForceStuckConflictForTesting(ctx, tlfID) 2085 require.NoError(t, err) 2086 2087 t.Log("User 1 updates mod time on a/b.") 2088 2089 mtime := time.Now() 2090 for i := 0; i < fastForwardRevThresh+2; i++ { 2091 mtime = mtime.Add(1 * time.Minute) 2092 err = kbfsOps1.SetMtime(ctx, fileB1, &mtime) 2093 require.NoError(t, err) 2094 err = kbfsOps1.SyncAll(ctx, fileB1.GetFolderBranch()) 2095 require.NoError(t, err) 2096 } 2097 2098 t.Log("Ensure only conflict files are in the conflict view") 2099 children, err := kbfsOps2.GetDirChildren(ctx, rootNode2) 2100 require.NoError(t, err) 2101 require.Len(t, children, 1+(maxConflictResolutionAttempts+1)) 2102 2103 // Expect updates for each of the unmerged revisions, plus exactly 2104 // one for the fast forward (but not more). However, if the 2105 // conflict file nodes haven't been garbage collected yet, we 2106 // might get 3x that (one batch for each sync op, one for each 2107 // remove op, and one for each resolution op), so allow some 2108 // wiggle room. This still isn't big enough to hold all of the 2109 // changes we'd get in a non-fast-forward though. 2110 numUpdatesExpected := 3*(maxConflictResolutionAttempts+1) + 1 2111 c := make(chan struct{}, numUpdatesExpected) 2112 cro := &testCRObserver{c, nil} 2113 err = config2.Notifier().RegisterForChanges( 2114 []data.FolderBranch{rootNode2.GetFolderBranch()}, cro) 2115 require.NoError(t, err) 2116 2117 t.Log("Clear the conflict state and re-enable CR.") 2118 err = kbfsOps2.ClearConflictView(ctx, tlfID) 2119 require.NoError(t, err) 2120 2121 t.Log("Ensure conflict files are gone") 2122 children, err = kbfsOps2.GetDirChildren(ctx, rootNode2) 2123 require.NoError(t, err) 2124 require.Len(t, children, 1) 2125 2126 ei, err := kbfsOps2.Stat(ctx, fileB2) 2127 require.NoError(t, err) 2128 require.Equal(t, mtime.UnixNano(), ei.Mtime) 2129 2130 err = kbfsOps2.SyncFromServer(ctx, rootNode2.GetFolderBranch(), nil) 2131 require.NoError(t, err) 2132} 2133