1package teams 2 3import ( 4 "context" 5 "fmt" 6 "sync" 7 "testing" 8 "time" 9 10 "github.com/keybase/client/go/sig3" 11 "github.com/keybase/client/go/teams/hidden" 12 13 "github.com/keybase/client/go/libkb" 14 "github.com/keybase/client/go/protocol/keybase1" 15 "github.com/stretchr/testify/require" 16) 17 18// See CORE-8860. We should be able to audit a stale team. That is, ever the merkle tree 19// is advertising a tail at 5, and we're only loaded through 3 (due to an unbusted cache), 20// the audit should still succeed. 21func TestAuditStaleTeam(t *testing.T) { 22 23 fus, tcs, cleanup := setupNTests(t, 5) 24 defer cleanup() 25 26 t.Logf("create team") 27 teamName, _ := createTeam2(*tcs[0]) 28 m := make([]libkb.MetaContext, 5) 29 for i, tc := range tcs { 30 m[i] = libkb.NewMetaContextForTest(*tc) 31 } 32 33 // We set up codenames for 3 users, A, B, C, D and E 34 const ( 35 A = 0 36 B = 1 37 C = 2 38 D = 3 39 E = 4 40 ) 41 42 t.Logf("A adds B to the team as an admin") 43 _, err := AddMember(m[0].Ctx(), tcs[0].G, teamName.String(), fus[1].Username, keybase1.TeamRole_ADMIN, nil) 44 require.NoError(t, err) 45 46 load := func(asUser int) { 47 _, err = Load(m[asUser].Ctx(), tcs[asUser].G, keybase1.LoadTeamArg{ 48 Name: teamName.String(), 49 Public: false, 50 StaleOK: true, 51 }) 52 require.NoError(t, err) 53 } 54 55 addC := func(asUser int) { 56 _, err = AddMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[C].Username, keybase1.TeamRole_READER, nil) 57 require.NoError(t, err) 58 } 59 60 addD := func(asUser int) { 61 _, err = AddMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[D].Username, keybase1.TeamRole_BOT, nil) 62 require.NoError(t, err) 63 } 64 65 addE := func(asUser int) { 66 _, err = AddMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[E].Username, keybase1.TeamRole_RESTRICTEDBOT, &keybase1.TeamBotSettings{}) 67 require.NoError(t, err) 68 } 69 70 rmC := func(asUser int) { 71 err = RemoveMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[C].Username) 72 require.NoError(t, err) 73 } 74 75 rmD := func(asUser int) { 76 err = RemoveMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[D].Username) 77 require.NoError(t, err) 78 } 79 80 rmE := func(asUser int) { 81 err = RemoveMember(m[asUser].Ctx(), tcs[asUser].G, teamName.String(), fus[E].Username) 82 require.NoError(t, err) 83 } 84 85 setFastAudits := func(m libkb.MetaContext) { 86 // do a lot of probes so we're likely to find issues 87 m.G().Env.Test.TeamAuditParams = &libkb.TeamAuditParams{ 88 NumPostProbes: 10, 89 MerkleMovementTrigger: keybase1.Seqno(1), 90 RootFreshness: time.Duration(1), 91 LRUSize: 500, 92 NumPreProbes: 3, 93 Parallelism: 3, 94 } 95 } 96 97 setSlowAudits := func(m libkb.MetaContext) { 98 m.G().Env.Test.TeamAuditParams = &libkb.TeamAuditParams{ 99 NumPostProbes: 1, 100 MerkleMovementTrigger: keybase1.Seqno(1000000), 101 RootFreshness: time.Hour, 102 LRUSize: 500, 103 NumPreProbes: 3, 104 Parallelism: 3, 105 } 106 } 107 108 // A adds C, D and E to the team and triggers an Audit 109 setFastAudits(m[A]) 110 addC(A) 111 addD(A) 112 addE(A) 113 114 // A removes C, D and E from the team, and loads the team, but does *not* trigger an audit 115 setSlowAudits(m[A]) 116 rmC(A) 117 rmD(A) 118 rmE(A) 119 load(A) 120 121 t.Logf("User B rotates the key a bunch of times") 122 123 // B rotates the key by adding and remove C a bunch of times. 124 for i := 0; i < 3; i++ { 125 addC(B) 126 rmC(B) 127 } 128 129 // A updates to the latest merkle root. 130 _, err = tcs[A].G.MerkleClient.FetchRootFromServer(m[A], 0) 131 require.NoError(t, err) 132 133 // A forces an audit on a stale team. 134 setFastAudits(m[A]) 135 t.Logf("User A loading the team, and auditing on an primed cached") 136 load(A) 137} 138 139func TestAuditRotateAudit(t *testing.T) { 140 fus, tcs, cleanup := setupNTests(t, 4) 141 defer cleanup() 142 143 t.Logf("create team") 144 teamName, teamID := createTeam2(*tcs[0]) 145 m := make([]libkb.MetaContext, 4) 146 for i, tc := range tcs { 147 m[i] = libkb.NewMetaContextForTest(*tc) 148 } 149 150 // We set up codenames for 3 users, A, B, C, and D 151 const ( 152 A = 0 153 B = 1 154 C = 2 155 D = 3 156 ) 157 158 load := func() { 159 _, err := Load(m[A].Ctx(), tcs[A].G, keybase1.LoadTeamArg{ 160 Name: teamName.String(), 161 Public: false, 162 ForceRepoll: true, 163 }) 164 require.NoError(t, err) 165 } 166 167 addB := func() keybase1.Seqno { 168 _, err := AddMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[B].Username, keybase1.TeamRole_READER, nil) 169 require.NoError(t, err) 170 return 1 171 } 172 173 addC := func() keybase1.Seqno { 174 _, err := AddMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[C].Username, keybase1.TeamRole_BOT, nil) 175 require.NoError(t, err) 176 return 1 177 } 178 179 addD := func() keybase1.Seqno { 180 _, err := AddMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[D].Username, keybase1.TeamRole_RESTRICTEDBOT, &keybase1.TeamBotSettings{}) 181 require.NoError(t, err) 182 // adding a RESTRICTEDBOT adds an additional bot_settings link 183 return 2 184 } 185 186 rmB := func() keybase1.Seqno { 187 err := RemoveMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[B].Username) 188 require.NoError(t, err) 189 return 1 190 } 191 192 rmC := func() keybase1.Seqno { 193 err := RemoveMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[C].Username) 194 require.NoError(t, err) 195 return 1 196 } 197 198 rmD := func() keybase1.Seqno { 199 err := RemoveMember(m[A].Ctx(), tcs[A].G, teamName.String(), fus[D].Username) 200 require.NoError(t, err) 201 return 1 202 } 203 204 setFastAudits := func() { 205 // do a lot of probes so we're likely to find issues 206 m[A].G().Env.Test.TeamAuditParams = &libkb.TeamAuditParams{ 207 NumPostProbes: 10, 208 MerkleMovementTrigger: keybase1.Seqno(1), 209 RootFreshness: time.Duration(1), 210 LRUSize: 500, 211 NumPreProbes: 3, 212 Parallelism: 3, 213 } 214 } 215 216 assertAuditTo := func(n keybase1.Seqno) { 217 auditor := m[A].G().GetTeamAuditor().(*Auditor) 218 history, err := auditor.getFromCache(m[A], teamID, auditor.getLRU()) 219 require.NoError(t, err) 220 require.Equal(t, n, lastAudit(history).MaxChainSeqno) 221 } 222 223 setFastAudits() 224 actions := []func() keybase1.Seqno{addB, rmB, addC, rmC, addD, rmD} 225 expectedSeqno := keybase1.Seqno(1) 226 for _, action := range actions { 227 expectedSeqno += action() 228 load() 229 assertAuditTo(expectedSeqno) 230 } 231} 232 233type CorruptingMerkleClient struct { 234 libkb.MerkleClientInterface 235 236 corruptor func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) 237} 238 239func (c CorruptingMerkleClient) LookupLeafAtSeqnoForAudit(m libkb.MetaContext, leafID keybase1.UserOrTeamID, s keybase1.Seqno, processHiddenResponseFunc libkb.ProcessHiddenRespFunc) (leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) { 240 return c.corruptor(c.MerkleClientInterface.LookupLeafAtSeqnoForAudit(m, leafID, s, processHiddenResponseFunc)) 241} 242 243var _ libkb.MerkleClientInterface = CorruptingMerkleClient{} 244 245type MockRandom struct { 246 libkb.SecureRandom 247 nextOutputs []int64 248 t *testing.T 249} 250 251func (m *MockRandom) RndRange(lo, hi int64) (int64, error) { 252 // pop and return the first output which is appropriate 253 for i, n := range m.nextOutputs { 254 if lo <= n && n <= hi { 255 m.nextOutputs = append(m.nextOutputs[0:i], m.nextOutputs[i+1:]...) 256 m.t.Logf("MockRandom: Output %v in range %v,%v (have %v left)", n, lo, hi, m.nextOutputs) 257 return n, nil 258 } 259 } 260 return 0, fmt.Errorf("MockRandom: output not found in range %v,%v (have %v left)", lo, hi, m.nextOutputs) 261} 262 263var _ libkb.Random = (*MockRandom)(nil) 264 265func TestAuditFailsIfDataIsInconsistent(t *testing.T) { 266 fus, tcs, cleanup := setupNTests(t, 3) 267 defer cleanup() 268 269 t.Logf("create team") 270 teamName, teamID := createTeam2(*tcs[0]) 271 m := make([]libkb.MetaContext, 3) 272 for i, tc := range tcs { 273 m[i] = libkb.NewMetaContextForTest(*tc) 274 } 275 276 // We set up codenames for 3 users, A, B, C 277 const ( 278 A = 0 279 B = 1 280 C = 2 281 ) 282 283 add := func(adder, addee int) keybase1.Seqno { 284 _, err := AddMember(m[adder].Ctx(), tcs[adder].G, teamName.String(), fus[addee].Username, keybase1.TeamRole_READER, nil) 285 require.NoError(t, err) 286 return 1 287 } 288 289 setAudits := func(user int) { 290 m[user].G().Env.Test.TeamAuditParams = &libkb.TeamAuditParams{ 291 NumPostProbes: 3, 292 MerkleMovementTrigger: keybase1.Seqno(1), 293 RootFreshness: time.Duration(1), 294 LRUSize: 500, 295 NumPreProbes: 3, 296 Parallelism: 3, 297 } 298 } 299 300 assertAuditTo := func(user int, mainSeqno, hiddenSeqno keybase1.Seqno) { 301 auditor := m[user].G().GetTeamAuditor().(*Auditor) 302 history, err := auditor.getFromCache(m[user], teamID, auditor.getLRU()) 303 require.NoError(t, err) 304 require.Equal(t, mainSeqno, lastAudit(history).MaxChainSeqno) 305 require.Equal(t, hiddenSeqno, lastAudit(history).MaxHiddenSeqno) 306 } 307 308 setAudits(B) 309 310 // A adds B to the team 311 add(A, B) 312 313 makeHiddenRotation(t, m[A].G(), teamName) 314 requestNewBlindTreeFromArchitectAndWaitUntilDone(t, tcs[A]) 315 makeHiddenRotation(t, m[A].G(), teamName) 316 requestNewBlindTreeFromArchitectAndWaitUntilDone(t, tcs[A]) 317 add(A, C) 318 319 team, err := GetForTestByStringName(context.TODO(), m[A].G(), teamName.String()) 320 require.NoError(t, err) 321 322 headMerkleSeqno := int64(team.MainChain().Chain.HeadMerkle.Seqno) 323 t.Logf("headMerkleSeqno: %v", headMerkleSeqno) 324 325 firstWithHiddenS, err := m[A].G().GetMerkleClient().FirstMainRootWithHiddenRootHash(m[A]) 326 require.NoError(t, err) 327 firstWithHidden := int64(firstWithHiddenS) 328 t.Logf("firstWithHidden: %v", firstWithHidden) 329 root := m[A].G().GetMerkleClient().LastRoot(m[A]) 330 require.NotNil(t, root) 331 high := int64(*root.Seqno()) 332 t.Logf("latest root: %v %X", root.Seqno(), root.HashMeta()) 333 334 for i := headMerkleSeqno; i <= high; i++ { 335 leaf, _, hiddenResp, err := m[B].G().GetMerkleClient().LookupLeafAtSeqnoForAudit(m[B], teamID.AsUserOrTeam(), keybase1.Seqno(i), hidden.ProcessHiddenResponseFunc) 336 require.NoError(t, err) 337 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 338 t.Logf("Seqno %v Leaf %v Hidden %v", i, leaf.Private.Seqno, hiddenResp) 339 } else { 340 t.Logf("Seqno %v Leaf EMPTY Hidden %v", i, hiddenResp) 341 } 342 343 } 344 345 merkle := m[B].G().GetMerkleClient() 346 rand := m[B].G().GetRandom() 347 348 corruptMerkle := CorruptingMerkleClient{ 349 MerkleClientInterface: merkle, 350 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 351 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 352 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 353 leaf.Private.LinkID[0] ^= 0xff 354 t.Logf("Corruptor: altering LINKID for %v", leaf.Private.Seqno) 355 } 356 return leaf, root, hiddenResp, err 357 }, 358 } 359 m[B].G().SetMerkleClient(corruptMerkle) 360 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 361 362 auditor := m[B].G().GetTeamAuditor().(*Auditor) 363 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 364 require.Error(t, err) 365 require.IsType(t, AuditError{}, err) 366 require.Contains(t, err.Error(), "team chain linkID mismatch") 367 368 // repeat a second time to ensure that a failed audit is not cached (and thus skipped the second time) 369 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 370 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 371 require.Error(t, err) 372 require.IsType(t, AuditError{}, err) 373 require.Contains(t, err.Error(), "team chain linkID mismatch") 374 375 corruptMerkle = CorruptingMerkleClient{ 376 MerkleClientInterface: merkle, 377 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 378 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 379 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 380 leaf.Private.Seqno += 5 381 t.Logf("Corruptor: altering Seqno, leaf = %+v", leaf) 382 } 383 return leaf, root, hiddenResp, err 384 }, 385 } 386 m[B].G().SetMerkleClient(corruptMerkle) 387 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 388 389 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 390 require.Error(t, err) 391 require.IsType(t, AuditError{}, err) 392 require.Contains(t, err.Error(), "team chain rollback") 393 394 // now, let's try to mess with the preProbes, by making it appear as if the team existed before it was actually created. 395 corruptMerkle = CorruptingMerkleClient{ 396 MerkleClientInterface: merkle, 397 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 398 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 399 if leaf == nil { 400 leaf = &libkb.MerkleGenericLeaf{ 401 LeafID: teamID.AsUserOrTeam(), 402 } 403 } 404 if leaf.Private == nil { 405 t.Logf("Corruptor: creating a fake leaf when there should have been none") 406 leaf.Private = &libkb.MerkleTriple{ 407 Seqno: 4, 408 LinkID: []byte{0x00, 0x01, 0x02}, 409 } 410 } 411 return leaf, root, hiddenResp, err 412 }, 413 } 414 m[B].G().SetMerkleClient(corruptMerkle) 415 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 416 417 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 418 require.Error(t, err) 419 require.IsType(t, AuditError{}, err) 420 require.Contains(t, err.Error(), "merkle root should not have had a leaf for team") 421 422 // now, test that the server cannot cheat on the hidden chain. 423 team, err = GetForTestByStringName(context.TODO(), m[A].G(), teamName.String()) 424 require.NoError(t, err) 425 root = m[A].G().GetMerkleClient().LastRoot(m[A]) 426 require.NotNil(t, root) 427 428 corruptMerkle = CorruptingMerkleClient{ 429 MerkleClientInterface: merkle, 430 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 431 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 432 if hiddenResp.RespType == libkb.MerkleHiddenResponseTypeABSENCEPROOF { 433 t.Logf("Corruptor: creating a fake hidden leaf leaf when there should have been none") 434 hiddenResp.RespType = libkb.MerkleHiddenResponseTypeOK 435 hiddenResp.CommittedHiddenTail = &sig3.Tail{ 436 ChainType: keybase1.SeqType_TEAM_PRIVATE_HIDDEN, 437 Seqno: 5, 438 } 439 } 440 return leaf, root, hiddenResp, err 441 }, 442 } 443 m[B].G().SetMerkleClient(corruptMerkle) 444 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 445 446 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 447 require.Error(t, err) 448 require.IsType(t, AuditError{}, err) 449 require.Contains(t, err.Error(), "expected an ABSENCE PROOF") 450 451 corruptMerkle = CorruptingMerkleClient{ 452 MerkleClientInterface: merkle, 453 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 454 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 455 if hiddenResp.RespType == libkb.MerkleHiddenResponseTypeOK { 456 t.Logf("Corruptor: altering hidden seqno") 457 hiddenResp.CommittedHiddenTail.Seqno += 5 458 } 459 return leaf, root, hiddenResp, err 460 }, 461 } 462 m[B].G().SetMerkleClient(corruptMerkle) 463 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 464 465 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 466 require.Error(t, err) 467 require.IsType(t, AuditError{}, err) 468 require.Contains(t, err.Error(), "team hidden chain rollback") 469 470 corruptMerkle = CorruptingMerkleClient{ 471 MerkleClientInterface: merkle, 472 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 473 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 474 if hiddenResp.RespType == libkb.MerkleHiddenResponseTypeOK { 475 hiddenResp.CommittedHiddenTail.Hash[0] ^= 0xff 476 t.Logf("Corruptor: altering LINKID for hidden seqno %v", hiddenResp.CommittedHiddenTail.Seqno) 477 } 478 return leaf, root, hiddenResp, err 479 }, 480 } 481 m[B].G().SetMerkleClient(corruptMerkle) 482 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{firstWithHidden - 1, firstWithHidden, firstWithHidden + 1, headMerkleSeqno, headMerkleSeqno + 1, high - 1}}) 483 484 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 485 require.Error(t, err) 486 require.IsType(t, AuditError{}, err) 487 require.Contains(t, err.Error(), "hidden team chain linkID mismatch") 488 489 // with the original merkle client (i.e. when the server response is not altered), the audit should succeed 490 m[B].G().SetMerkleClient(merkle) 491 m[B].G().SetRandom(rand) 492 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 493 require.NoError(t, err) 494 assertAuditTo(B, 3, 2) 495} 496 497func TestFailedProbesAreRetried(t *testing.T) { 498 fus, tcs, cleanup := setupNTests(t, 2) 499 defer cleanup() 500 501 // We set up codenames for 2 users, A, B 502 const ( 503 A = 0 504 B = 1 505 ) 506 507 makePaperKey(t, tcs[A]) 508 makePaperKey(t, tcs[B]) 509 510 t.Logf("create team") 511 teamName, teamID := createTeam2(*tcs[A]) 512 m := make([]libkb.MetaContext, 3) 513 for i, tc := range tcs { 514 m[i] = libkb.NewMetaContextForTest(*tc) 515 } 516 517 add := func(adder, addee int) keybase1.Seqno { 518 _, err := AddMember(m[adder].Ctx(), tcs[adder].G, teamName.String(), fus[addee].Username, keybase1.TeamRole_READER, nil) 519 require.NoError(t, err) 520 return 1 521 } 522 523 setFastAudits := func(user int) { 524 m[user].G().Env.Test.TeamAuditParams = &libkb.TeamAuditParams{ 525 NumPostProbes: 2, 526 MerkleMovementTrigger: keybase1.Seqno(1), 527 RootFreshness: time.Duration(1), 528 LRUSize: 500, 529 NumPreProbes: 2, 530 Parallelism: 3, 531 } 532 } 533 534 setFastAudits(B) 535 536 // A adds B to the team 537 add(A, B) 538 539 // make some extra merkle tree versions 540 makePaperKey(t, tcs[A]) 541 makePaperKey(t, tcs[B]) 542 makePaperKey(t, tcs[A]) 543 makePaperKey(t, tcs[B]) 544 makePaperKey(t, tcs[A]) 545 makePaperKey(t, tcs[B]) 546 547 team, err := GetForTestByStringName(context.TODO(), m[A].G(), teamName.String()) 548 require.NoError(t, err) 549 root := m[A].G().GetMerkleClient().LastRoot(m[A]) 550 require.NotNil(t, root) 551 latestRootSeqno := *root.Seqno() 552 t.Logf("latest root: %v %X", root.Seqno(), root.HashMeta()) 553 headMerkleSeqno := team.MainChain().Chain.HeadMerkle.Seqno 554 t.Logf("headMerkleSeqno: %v", headMerkleSeqno) 555 firstWithHiddenS, err := m[A].G().GetMerkleClient().FirstMainRootWithHiddenRootHash(m[A]) 556 require.NoError(t, err) 557 firstWithHidden := firstWithHiddenS 558 t.Logf("firstWithHidden: %v", firstWithHidden) 559 560 for i := headMerkleSeqno; i <= latestRootSeqno; i++ { 561 leaf, _, hiddenResp, err := m[B].G().GetMerkleClient().LookupLeafAtSeqnoForAudit(m[B], teamID.AsUserOrTeam(), i, hidden.ProcessHiddenResponseFunc) 562 require.NoError(t, err) 563 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 564 t.Logf("Seqno %v Leaf %v Hidden %v", i, leaf.Private.Seqno, hiddenResp) 565 } else { 566 t.Logf("Seqno %v Leaf EMPTY Hidden %v", i, hiddenResp) 567 } 568 } 569 570 auditor := m[B].G().GetTeamAuditor().(*Auditor) 571 lru := auditor.getLRU() 572 573 // no audits yet, history is nil. 574 history, err := auditor.getFromCache(m[B], teamID, lru) 575 require.NoError(t, err) 576 require.Nil(t, history) 577 578 merkle := m[B].G().GetMerkleClient() 579 rand := m[B].G().GetRandom() 580 581 // first we corrupt postProbes and test that those are retried 582 corruptMerkle := CorruptingMerkleClient{ 583 MerkleClientInterface: merkle, 584 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 585 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 586 if *root.Seqno() > headMerkleSeqno { 587 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 588 leaf.Private.LinkID[0] ^= 0xff 589 t.Logf("Corruptor: altering LINKID for %v", leaf.Private.Seqno) 590 } else { 591 t.Logf("Corruptor: introducing leaf at roor %v", *root.Seqno()) 592 leaf = &libkb.MerkleGenericLeaf{ 593 LeafID: teamID.AsUserOrTeam(), 594 Private: &libkb.MerkleTriple{Seqno: keybase1.Seqno(100)}, 595 } 596 } 597 } 598 return leaf, root, hiddenResp, err 599 }, 600 } 601 m[B].G().SetMerkleClient(corruptMerkle) 602 // the first two are for preprobes, the last for post probes 603 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{int64(firstWithHidden) - 1, int64(firstWithHidden + 1), int64(headMerkleSeqno) + 1, int64(headMerkleSeqno) + 2}}) 604 605 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 606 require.Error(t, err) 607 require.IsType(t, AuditError{}, err) 608 609 history, err = auditor.getFromCache(m[B], teamID, lru) 610 require.NoError(t, err) 611 require.Len(t, history.PreProbesToRetry, 0) 612 require.Len(t, history.PostProbesToRetry, 2) 613 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+1) 614 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+2) 615 616 var probesToTestLock sync.Mutex 617 probesToTest := make(map[keybase1.Seqno]bool) 618 probesToTestLock.Lock() 619 probesToTest[headMerkleSeqno+1] = true 620 probesToTest[headMerkleSeqno+2] = true 621 numProbes := 2 622 probesToTestLock.Unlock() 623 624 corruptMerkle = CorruptingMerkleClient{ 625 MerkleClientInterface: merkle, 626 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 627 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 628 probeSeqno := *root.Seqno() 629 if probeSeqno > headMerkleSeqno { 630 if probesToTest[probeSeqno] { 631 t.Logf("PostProbes: Seqno %v was retried", probeSeqno) 632 probesToTestLock.Lock() 633 probesToTest[probeSeqno] = false 634 numProbes-- 635 probesToTestLock.Unlock() 636 } 637 if leaf != nil && leaf.Private != nil && len(leaf.Private.LinkID) > 0 { 638 leaf.Private.LinkID[0] ^= 0xff 639 t.Logf("Corruptor: altering LINKID for %v", leaf.Private.Seqno) 640 } else { 641 t.Logf("Corruptor: introducing leaf at roor %v", *root.Seqno()) 642 leaf = &libkb.MerkleGenericLeaf{ 643 LeafID: teamID.AsUserOrTeam(), 644 Private: &libkb.MerkleTriple{Seqno: keybase1.Seqno(100)}, 645 } 646 } 647 } 648 return leaf, root, hiddenResp, err 649 }, 650 } 651 m[B].G().SetMerkleClient(corruptMerkle) 652 // note that the postProbes we will sample now are different from the ones which we failed on the first time, so we can test we are actually retrying those. 653 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{int64(firstWithHidden) - 1, int64(firstWithHidden + 1), int64(headMerkleSeqno) + 3, int64(headMerkleSeqno) + 4}}) 654 655 // repeat a second time and make sure that we retry the same probes 656 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 657 require.Error(t, err) 658 require.IsType(t, AuditError{}, err) 659 require.Zero(t, numProbes, "not all probes were retried") 660 661 // now test the preprobes are saved and retried on failure 662 corruptMerkle = CorruptingMerkleClient{ 663 MerkleClientInterface: merkle, 664 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 665 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 666 if leaf == nil { 667 leaf = &libkb.MerkleGenericLeaf{ 668 LeafID: teamID.AsUserOrTeam(), 669 } 670 } 671 if leaf.Private == nil { 672 t.Logf("Corruptor: creating a fake leaf when there should have been none") 673 leaf.Private = &libkb.MerkleTriple{ 674 Seqno: 4, 675 LinkID: []byte{0x00, 0x01, 0x02}, 676 } 677 } 678 return leaf, root, hiddenResp, err 679 }, 680 } 681 m[B].G().SetMerkleClient(corruptMerkle) 682 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{int64(firstWithHidden) - 1, int64(firstWithHidden + 1), int64(headMerkleSeqno) + 3, int64(headMerkleSeqno) + 4}}) 683 684 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 685 require.Error(t, err) 686 require.IsType(t, AuditError{}, err) 687 require.Contains(t, err.Error(), "merkle root should not have had a leaf for team") 688 689 history, err = auditor.getFromCache(m[B], teamID, lru) 690 require.NoError(t, err) 691 require.Len(t, history.PreProbesToRetry, 2) 692 require.Contains(t, history.PreProbesToRetry, firstWithHidden-1) 693 require.Contains(t, history.PreProbesToRetry, firstWithHidden+1) 694 695 // the old failed postprobes are still in the cache 696 require.Len(t, history.PostProbesToRetry, 2) 697 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+1) 698 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+2) 699 700 probesToTest = make(map[keybase1.Seqno]bool) 701 probesToTestLock.Lock() 702 probesToTest[firstWithHidden-1] = true 703 probesToTest[firstWithHidden+1] = true 704 probesToTestLock.Unlock() 705 numProbes = 2 706 707 corruptMerkle = CorruptingMerkleClient{ 708 MerkleClientInterface: merkle, 709 corruptor: func(leaf *libkb.MerkleGenericLeaf, root *libkb.MerkleRoot, hiddenResp *libkb.MerkleHiddenResponse, err error) (*libkb.MerkleGenericLeaf, *libkb.MerkleRoot, *libkb.MerkleHiddenResponse, error) { 710 t.Logf("Corruptor: received %v,%v,%v,%v", leaf, root, hiddenResp, err) 711 probeSeqno := *root.Seqno() 712 if probeSeqno <= headMerkleSeqno { 713 if probesToTest[probeSeqno] { 714 t.Logf("PreProbes: Seqno %v was retried", probeSeqno) 715 probesToTestLock.Lock() 716 probesToTest[probeSeqno] = false 717 numProbes-- 718 probesToTestLock.Unlock() 719 } 720 if leaf == nil { 721 leaf = &libkb.MerkleGenericLeaf{ 722 LeafID: teamID.AsUserOrTeam(), 723 } 724 } 725 if leaf.Private == nil { 726 t.Logf("Corruptor: creating a fake leaf when there should have been none") 727 leaf.Private = &libkb.MerkleTriple{ 728 Seqno: 4, 729 LinkID: []byte{0x00, 0x01, 0x02}, 730 } 731 } 732 } 733 return leaf, root, hiddenResp, err 734 }, 735 } 736 m[B].G().SetMerkleClient(corruptMerkle) 737 m[B].G().SetRandom(&MockRandom{t: t, nextOutputs: []int64{int64(firstWithHidden) - 2, int64(firstWithHidden) + 2, int64(headMerkleSeqno) + 3, int64(headMerkleSeqno) + 4}}) 738 739 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 740 require.Error(t, err) 741 require.IsType(t, AuditError{}, err) 742 require.Zero(t, numProbes, "not all probes were retried") 743 744 history, err = auditor.getFromCache(m[B], teamID, lru) 745 require.NoError(t, err) 746 require.Len(t, history.PreProbesToRetry, 2) 747 require.Contains(t, history.PreProbesToRetry, firstWithHidden-1) 748 require.Contains(t, history.PreProbesToRetry, firstWithHidden+1) 749 750 // the old failed postprobes are still in the cache 751 require.Len(t, history.PostProbesToRetry, 2) 752 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+1) 753 require.Contains(t, history.PostProbesToRetry, headMerkleSeqno+2) 754 755 // now stop any corruption and ensure the audit succeeds 756 m[B].G().SetMerkleClient(merkle) 757 m[B].G().SetRandom(rand) 758 err = auditor.AuditTeam(m[B], teamID, false, team.MainChain().Chain.HeadMerkle.Seqno, team.MainChain().Chain.LinkIDs, team.HiddenChain().GetOuter(), team.MainChain().Chain.LastSeqno, team.HiddenChain().GetLastCommittedSeqno(), root, keybase1.AuditMode_STANDARD) 759 require.NoError(t, err) 760} 761