1package txn_test 2 3import ( 4 "flag" 5 "fmt" 6 "sync" 7 "testing" 8 "time" 9 10 . "gopkg.in/check.v1" 11 "gopkg.in/mgo.v2" 12 "gopkg.in/mgo.v2/bson" 13 "gopkg.in/mgo.v2/dbtest" 14 "gopkg.in/mgo.v2/txn" 15) 16 17func TestAll(t *testing.T) { 18 TestingT(t) 19} 20 21type S struct { 22 server dbtest.DBServer 23 session *mgo.Session 24 db *mgo.Database 25 tc, sc *mgo.Collection 26 accounts *mgo.Collection 27 runner *txn.Runner 28} 29 30var _ = Suite(&S{}) 31 32type M map[string]interface{} 33 34func (s *S) SetUpSuite(c *C) { 35 s.server.SetPath(c.MkDir()) 36} 37 38func (s *S) TearDownSuite(c *C) { 39 s.server.Stop() 40} 41 42func (s *S) SetUpTest(c *C) { 43 s.server.Wipe() 44 45 txn.SetChaos(txn.Chaos{}) 46 txn.SetLogger(c) 47 txn.SetDebug(true) 48 49 s.session = s.server.Session() 50 s.db = s.session.DB("test") 51 s.tc = s.db.C("tc") 52 s.sc = s.db.C("tc.stash") 53 s.accounts = s.db.C("accounts") 54 s.runner = txn.NewRunner(s.tc) 55} 56 57func (s *S) TearDownTest(c *C) { 58 txn.SetLogger(nil) 59 txn.SetDebug(false) 60 s.session.Close() 61} 62 63type Account struct { 64 Id int `bson:"_id"` 65 Balance int 66} 67 68func (s *S) TestDocExists(c *C) { 69 err := s.accounts.Insert(M{"_id": 0, "balance": 300}) 70 c.Assert(err, IsNil) 71 72 exists := []txn.Op{{ 73 C: "accounts", 74 Id: 0, 75 Assert: txn.DocExists, 76 }} 77 missing := []txn.Op{{ 78 C: "accounts", 79 Id: 0, 80 Assert: txn.DocMissing, 81 }} 82 83 err = s.runner.Run(exists, "", nil) 84 c.Assert(err, IsNil) 85 err = s.runner.Run(missing, "", nil) 86 c.Assert(err, Equals, txn.ErrAborted) 87 88 err = s.accounts.RemoveId(0) 89 c.Assert(err, IsNil) 90 91 err = s.runner.Run(exists, "", nil) 92 c.Assert(err, Equals, txn.ErrAborted) 93 err = s.runner.Run(missing, "", nil) 94 c.Assert(err, IsNil) 95} 96 97func (s *S) TestInsert(c *C) { 98 err := s.accounts.Insert(M{"_id": 0, "balance": 300}) 99 c.Assert(err, IsNil) 100 101 ops := []txn.Op{{ 102 C: "accounts", 103 Id: 0, 104 Insert: M{"balance": 200}, 105 }} 106 107 err = s.runner.Run(ops, "", nil) 108 c.Assert(err, IsNil) 109 110 var account Account 111 err = s.accounts.FindId(0).One(&account) 112 c.Assert(err, IsNil) 113 c.Assert(account.Balance, Equals, 300) 114 115 ops[0].Id = 1 116 err = s.runner.Run(ops, "", nil) 117 c.Assert(err, IsNil) 118 119 err = s.accounts.FindId(1).One(&account) 120 c.Assert(err, IsNil) 121 c.Assert(account.Balance, Equals, 200) 122} 123 124func (s *S) TestInsertStructID(c *C) { 125 type id struct { 126 FirstName string 127 LastName string 128 } 129 ops := []txn.Op{{ 130 C: "accounts", 131 Id: id{FirstName: "John", LastName: "Jones"}, 132 Assert: txn.DocMissing, 133 Insert: M{"balance": 200}, 134 }, { 135 C: "accounts", 136 Id: id{FirstName: "Sally", LastName: "Smith"}, 137 Assert: txn.DocMissing, 138 Insert: M{"balance": 800}, 139 }} 140 141 err := s.runner.Run(ops, "", nil) 142 c.Assert(err, IsNil) 143 144 n, err := s.accounts.Find(nil).Count() 145 c.Assert(err, IsNil) 146 c.Assert(n, Equals, 2) 147} 148 149func (s *S) TestRemove(c *C) { 150 err := s.accounts.Insert(M{"_id": 0, "balance": 300}) 151 c.Assert(err, IsNil) 152 153 ops := []txn.Op{{ 154 C: "accounts", 155 Id: 0, 156 Remove: true, 157 }} 158 159 err = s.runner.Run(ops, "", nil) 160 c.Assert(err, IsNil) 161 162 err = s.accounts.FindId(0).One(nil) 163 c.Assert(err, Equals, mgo.ErrNotFound) 164 165 err = s.runner.Run(ops, "", nil) 166 c.Assert(err, IsNil) 167} 168 169func (s *S) TestUpdate(c *C) { 170 var err error 171 err = s.accounts.Insert(M{"_id": 0, "balance": 200}) 172 c.Assert(err, IsNil) 173 err = s.accounts.Insert(M{"_id": 1, "balance": 200}) 174 c.Assert(err, IsNil) 175 176 ops := []txn.Op{{ 177 C: "accounts", 178 Id: 0, 179 Update: M{"$inc": M{"balance": 100}}, 180 }} 181 182 err = s.runner.Run(ops, "", nil) 183 c.Assert(err, IsNil) 184 185 var account Account 186 err = s.accounts.FindId(0).One(&account) 187 c.Assert(err, IsNil) 188 c.Assert(account.Balance, Equals, 300) 189 190 ops[0].Id = 1 191 192 err = s.accounts.FindId(1).One(&account) 193 c.Assert(err, IsNil) 194 c.Assert(account.Balance, Equals, 200) 195} 196 197func (s *S) TestInsertUpdate(c *C) { 198 ops := []txn.Op{{ 199 C: "accounts", 200 Id: 0, 201 Insert: M{"_id": 0, "balance": 200}, 202 }, { 203 C: "accounts", 204 Id: 0, 205 Update: M{"$inc": M{"balance": 100}}, 206 }} 207 208 err := s.runner.Run(ops, "", nil) 209 c.Assert(err, IsNil) 210 211 var account Account 212 err = s.accounts.FindId(0).One(&account) 213 c.Assert(err, IsNil) 214 c.Assert(account.Balance, Equals, 300) 215 216 err = s.runner.Run(ops, "", nil) 217 c.Assert(err, IsNil) 218 219 err = s.accounts.FindId(0).One(&account) 220 c.Assert(err, IsNil) 221 c.Assert(account.Balance, Equals, 400) 222} 223 224func (s *S) TestUpdateInsert(c *C) { 225 ops := []txn.Op{{ 226 C: "accounts", 227 Id: 0, 228 Update: M{"$inc": M{"balance": 100}}, 229 }, { 230 C: "accounts", 231 Id: 0, 232 Insert: M{"_id": 0, "balance": 200}, 233 }} 234 235 err := s.runner.Run(ops, "", nil) 236 c.Assert(err, IsNil) 237 238 var account Account 239 err = s.accounts.FindId(0).One(&account) 240 c.Assert(err, IsNil) 241 c.Assert(account.Balance, Equals, 200) 242 243 err = s.runner.Run(ops, "", nil) 244 c.Assert(err, IsNil) 245 246 err = s.accounts.FindId(0).One(&account) 247 c.Assert(err, IsNil) 248 c.Assert(account.Balance, Equals, 300) 249} 250 251func (s *S) TestInsertRemoveInsert(c *C) { 252 ops := []txn.Op{{ 253 C: "accounts", 254 Id: 0, 255 Insert: M{"_id": 0, "balance": 200}, 256 }, { 257 C: "accounts", 258 Id: 0, 259 Remove: true, 260 }, { 261 C: "accounts", 262 Id: 0, 263 Insert: M{"_id": 0, "balance": 300}, 264 }} 265 266 err := s.runner.Run(ops, "", nil) 267 c.Assert(err, IsNil) 268 269 var account Account 270 err = s.accounts.FindId(0).One(&account) 271 c.Assert(err, IsNil) 272 c.Assert(account.Balance, Equals, 300) 273} 274 275func (s *S) TestQueueStashing(c *C) { 276 txn.SetChaos(txn.Chaos{ 277 KillChance: 1, 278 Breakpoint: "set-applying", 279 }) 280 281 opses := [][]txn.Op{{{ 282 C: "accounts", 283 Id: 0, 284 Insert: M{"balance": 100}, 285 }}, {{ 286 C: "accounts", 287 Id: 0, 288 Remove: true, 289 }}, {{ 290 C: "accounts", 291 Id: 0, 292 Insert: M{"balance": 200}, 293 }}, {{ 294 C: "accounts", 295 Id: 0, 296 Update: M{"$inc": M{"balance": 100}}, 297 }}} 298 299 var last bson.ObjectId 300 for _, ops := range opses { 301 last = bson.NewObjectId() 302 err := s.runner.Run(ops, last, nil) 303 c.Assert(err, Equals, txn.ErrChaos) 304 } 305 306 txn.SetChaos(txn.Chaos{}) 307 err := s.runner.Resume(last) 308 c.Assert(err, IsNil) 309 310 var account Account 311 err = s.accounts.FindId(0).One(&account) 312 c.Assert(err, IsNil) 313 c.Assert(account.Balance, Equals, 300) 314} 315 316func (s *S) TestInfo(c *C) { 317 ops := []txn.Op{{ 318 C: "accounts", 319 Id: 0, 320 Assert: txn.DocMissing, 321 }} 322 323 id := bson.NewObjectId() 324 err := s.runner.Run(ops, id, M{"n": 42}) 325 c.Assert(err, IsNil) 326 327 var t struct{ I struct{ N int } } 328 err = s.tc.FindId(id).One(&t) 329 c.Assert(err, IsNil) 330 c.Assert(t.I.N, Equals, 42) 331} 332 333func (s *S) TestErrors(c *C) { 334 doc := bson.M{"foo": 1} 335 tests := []txn.Op{{ 336 C: "c", 337 Id: 0, 338 }, { 339 C: "c", 340 Id: 0, 341 Insert: doc, 342 Remove: true, 343 }, { 344 C: "c", 345 Id: 0, 346 Insert: doc, 347 Update: doc, 348 }, { 349 C: "c", 350 Id: 0, 351 Update: doc, 352 Remove: true, 353 }, { 354 C: "c", 355 Assert: doc, 356 }, { 357 Id: 0, 358 Assert: doc, 359 }} 360 361 txn.SetChaos(txn.Chaos{KillChance: 1.0}) 362 for _, op := range tests { 363 c.Logf("op: %v", op) 364 err := s.runner.Run([]txn.Op{op}, "", nil) 365 c.Assert(err, ErrorMatches, "error in transaction op 0: .*") 366 } 367} 368 369func (s *S) TestAssertNestedOr(c *C) { 370 // Assert uses $or internally. Ensure nesting works. 371 err := s.accounts.Insert(M{"_id": 0, "balance": 300}) 372 c.Assert(err, IsNil) 373 374 ops := []txn.Op{{ 375 C: "accounts", 376 Id: 0, 377 Assert: bson.D{{"$or", []bson.D{{{"balance", 100}}, {{"balance", 300}}}}}, 378 Update: bson.D{{"$inc", bson.D{{"balance", 100}}}}, 379 }} 380 381 err = s.runner.Run(ops, "", nil) 382 c.Assert(err, IsNil) 383 384 var account Account 385 err = s.accounts.FindId(0).One(&account) 386 c.Assert(err, IsNil) 387 c.Assert(account.Balance, Equals, 400) 388} 389 390func (s *S) TestVerifyFieldOrdering(c *C) { 391 // Used to have a map in certain operations, which means 392 // the ordering of fields would be messed up. 393 fields := bson.D{{"a", 1}, {"b", 2}, {"c", 3}} 394 ops := []txn.Op{{ 395 C: "accounts", 396 Id: 0, 397 Insert: fields, 398 }} 399 400 err := s.runner.Run(ops, "", nil) 401 c.Assert(err, IsNil) 402 403 var d bson.D 404 err = s.accounts.FindId(0).One(&d) 405 c.Assert(err, IsNil) 406 407 var filtered bson.D 408 for _, e := range d { 409 switch e.Name { 410 case "a", "b", "c": 411 filtered = append(filtered, e) 412 } 413 } 414 c.Assert(filtered, DeepEquals, fields) 415} 416 417func (s *S) TestChangeLog(c *C) { 418 chglog := s.db.C("chglog") 419 s.runner.ChangeLog(chglog) 420 421 ops := []txn.Op{{ 422 C: "debts", 423 Id: 0, 424 Assert: txn.DocMissing, 425 }, { 426 C: "accounts", 427 Id: 0, 428 Insert: M{"balance": 300}, 429 }, { 430 C: "accounts", 431 Id: 1, 432 Insert: M{"balance": 300}, 433 }, { 434 C: "people", 435 Id: "joe", 436 Insert: M{"accounts": []int64{0, 1}}, 437 }} 438 id := bson.NewObjectId() 439 err := s.runner.Run(ops, id, nil) 440 c.Assert(err, IsNil) 441 442 type IdList []interface{} 443 type Log struct { 444 Docs IdList "d" 445 Revnos []int64 "r" 446 } 447 var m map[string]*Log 448 err = chglog.FindId(id).One(&m) 449 c.Assert(err, IsNil) 450 451 c.Assert(m["accounts"], DeepEquals, &Log{IdList{0, 1}, []int64{2, 2}}) 452 c.Assert(m["people"], DeepEquals, &Log{IdList{"joe"}, []int64{2}}) 453 c.Assert(m["debts"], IsNil) 454 455 ops = []txn.Op{{ 456 C: "accounts", 457 Id: 0, 458 Update: M{"$inc": M{"balance": 100}}, 459 }, { 460 C: "accounts", 461 Id: 1, 462 Update: M{"$inc": M{"balance": 100}}, 463 }} 464 id = bson.NewObjectId() 465 err = s.runner.Run(ops, id, nil) 466 c.Assert(err, IsNil) 467 468 m = nil 469 err = chglog.FindId(id).One(&m) 470 c.Assert(err, IsNil) 471 472 c.Assert(m["accounts"], DeepEquals, &Log{IdList{0, 1}, []int64{3, 3}}) 473 c.Assert(m["people"], IsNil) 474 475 ops = []txn.Op{{ 476 C: "accounts", 477 Id: 0, 478 Remove: true, 479 }, { 480 C: "people", 481 Id: "joe", 482 Remove: true, 483 }} 484 id = bson.NewObjectId() 485 err = s.runner.Run(ops, id, nil) 486 c.Assert(err, IsNil) 487 488 m = nil 489 err = chglog.FindId(id).One(&m) 490 c.Assert(err, IsNil) 491 492 c.Assert(m["accounts"], DeepEquals, &Log{IdList{0}, []int64{-4}}) 493 c.Assert(m["people"], DeepEquals, &Log{IdList{"joe"}, []int64{-3}}) 494} 495 496func (s *S) TestPurgeMissing(c *C) { 497 txn.SetChaos(txn.Chaos{ 498 KillChance: 1, 499 Breakpoint: "set-applying", 500 }) 501 502 err := s.accounts.Insert(M{"_id": 0, "balance": 100}) 503 c.Assert(err, IsNil) 504 err = s.accounts.Insert(M{"_id": 1, "balance": 100}) 505 c.Assert(err, IsNil) 506 507 ops1 := []txn.Op{{ 508 C: "accounts", 509 Id: 3, 510 Insert: M{"balance": 100}, 511 }} 512 513 ops2 := []txn.Op{{ 514 C: "accounts", 515 Id: 0, 516 Remove: true, 517 }, { 518 C: "accounts", 519 Id: 1, 520 Update: M{"$inc": M{"balance": 100}}, 521 }, { 522 C: "accounts", 523 Id: 2, 524 Insert: M{"balance": 100}, 525 }} 526 527 first := bson.NewObjectId() 528 c.Logf("---- Running ops1 under transaction %q, to be canceled by chaos", first.Hex()) 529 err = s.runner.Run(ops1, first, nil) 530 c.Assert(err, Equals, txn.ErrChaos) 531 532 last := bson.NewObjectId() 533 c.Logf("---- Running ops2 under transaction %q, to be canceled by chaos", last.Hex()) 534 err = s.runner.Run(ops2, last, nil) 535 c.Assert(err, Equals, txn.ErrChaos) 536 537 c.Logf("---- Removing transaction %q", last.Hex()) 538 err = s.tc.RemoveId(last) 539 c.Assert(err, IsNil) 540 541 c.Logf("---- Disabling chaos and attempting to resume all") 542 txn.SetChaos(txn.Chaos{}) 543 err = s.runner.ResumeAll() 544 c.Assert(err, IsNil) 545 546 again := bson.NewObjectId() 547 c.Logf("---- Running ops2 again under transaction %q, to fail for missing transaction", again.Hex()) 548 err = s.runner.Run(ops2, again, nil) 549 c.Assert(err, ErrorMatches, "cannot find transaction .*") 550 551 c.Logf("---- Purging missing transactions") 552 err = s.runner.PurgeMissing("accounts") 553 c.Assert(err, IsNil) 554 555 c.Logf("---- Resuming pending transactions") 556 err = s.runner.ResumeAll() 557 c.Assert(err, IsNil) 558 559 expect := []struct{ Id, Balance int }{ 560 {0, -1}, 561 {1, 200}, 562 {2, 100}, 563 {3, 100}, 564 } 565 var got Account 566 for _, want := range expect { 567 err = s.accounts.FindId(want.Id).One(&got) 568 if want.Balance == -1 { 569 if err != mgo.ErrNotFound { 570 c.Errorf("Account %d should not exist, find got err=%#v", err) 571 } 572 } else if err != nil { 573 c.Errorf("Account %d should have balance of %d, but wasn't found", want.Id, want.Balance) 574 } else if got.Balance != want.Balance { 575 c.Errorf("Account %d should have balance of %d, got %d", want.Id, want.Balance, got.Balance) 576 } 577 } 578} 579 580func (s *S) TestTxnQueueStashStressTest(c *C) { 581 txn.SetChaos(txn.Chaos{ 582 SlowdownChance: 0.3, 583 Slowdown: 50 * time.Millisecond, 584 }) 585 defer txn.SetChaos(txn.Chaos{}) 586 587 // So we can run more iterations of the test in less time. 588 txn.SetDebug(false) 589 590 const runners = 10 591 const inserts = 10 592 const repeat = 100 593 594 for r := 0; r < repeat; r++ { 595 var wg sync.WaitGroup 596 wg.Add(runners) 597 for i := 0; i < runners; i++ { 598 go func(i, r int) { 599 defer wg.Done() 600 601 session := s.session.New() 602 defer session.Close() 603 runner := txn.NewRunner(s.tc.With(session)) 604 605 for j := 0; j < inserts; j++ { 606 ops := []txn.Op{{ 607 C: "accounts", 608 Id: fmt.Sprintf("insert-%d-%d", r, j), 609 Insert: bson.M{ 610 "added-by": i, 611 }, 612 }} 613 err := runner.Run(ops, "", nil) 614 if err != txn.ErrAborted { 615 c.Check(err, IsNil) 616 } 617 } 618 }(i, r) 619 } 620 wg.Wait() 621 } 622} 623 624func (s *S) TestPurgeMissingPipelineSizeLimit(c *C) { 625 // This test ensures that PurgeMissing can handle very large 626 // txn-queue fields. Previous iterations of PurgeMissing would 627 // trigger a 16MB aggregation pipeline result size limit when run 628 // against a documents or stashes with large numbers of txn-queue 629 // entries. PurgeMissing now no longer uses aggregation pipelines 630 // to work around this limit. 631 632 // The pipeline result size limitation was removed from MongoDB in 2.6 so 633 // this test is only run for older MongoDB version. 634 build, err := s.session.BuildInfo() 635 c.Assert(err, IsNil) 636 if build.VersionAtLeast(2, 6) { 637 c.Skip("This tests a problem that can only happen with MongoDB < 2.6 ") 638 } 639 640 // Insert a single document to work with. 641 err = s.accounts.Insert(M{"_id": 0, "balance": 100}) 642 c.Assert(err, IsNil) 643 644 ops := []txn.Op{{ 645 C: "accounts", 646 Id: 0, 647 Update: M{"$inc": M{"balance": 100}}, 648 }} 649 650 // Generate one successful transaction. 651 good := bson.NewObjectId() 652 c.Logf("---- Running ops under transaction %q", good.Hex()) 653 err = s.runner.Run(ops, good, nil) 654 c.Assert(err, IsNil) 655 656 // Generate another transaction which which will go missing. 657 missing := bson.NewObjectId() 658 c.Logf("---- Running ops under transaction %q (which will go missing)", missing.Hex()) 659 err = s.runner.Run(ops, missing, nil) 660 c.Assert(err, IsNil) 661 662 err = s.tc.RemoveId(missing) 663 c.Assert(err, IsNil) 664 665 // Generate a txn-queue on the test document that's large enough 666 // that it used to cause PurgeMissing to exceed MongoDB's pipeline 667 // result 16MB size limit (MongoDB 2.4 and older only). 668 // 669 // The contents of the txn-queue field doesn't matter, only that 670 // it's big enough to trigger the size limit. The required size 671 // can also be achieved by using multiple documents as long as the 672 // cumulative size of all the txn-queue fields exceeds the 673 // pipeline limit. A single document is easier to work with for 674 // this test however. 675 // 676 // The txn id of the successful transaction is used fill the 677 // txn-queue because this takes advantage of a short circuit in 678 // PurgeMissing, dramatically speeding up the test run time. 679 const fakeQueueLen = 250000 680 fakeTxnQueue := make([]string, fakeQueueLen) 681 token := good.Hex() + "_12345678" // txn id + nonce 682 for i := 0; i < fakeQueueLen; i++ { 683 fakeTxnQueue[i] = token 684 } 685 686 err = s.accounts.UpdateId(0, bson.M{ 687 "$set": bson.M{"txn-queue": fakeTxnQueue}, 688 }) 689 c.Assert(err, IsNil) 690 691 // PurgeMissing could hit the same pipeline result size limit when 692 // processing the txn-queue fields of stash documents so insert 693 // the large txn-queue there too to ensure that no longer happens. 694 err = s.sc.Insert( 695 bson.D{{"c", "accounts"}, {"id", 0}}, 696 bson.M{"txn-queue": fakeTxnQueue}, 697 ) 698 c.Assert(err, IsNil) 699 700 c.Logf("---- Purging missing transactions") 701 err = s.runner.PurgeMissing("accounts") 702 c.Assert(err, IsNil) 703} 704 705var flaky = flag.Bool("flaky", false, "Include flaky tests") 706 707func (s *S) TestTxnQueueStressTest(c *C) { 708 // This fails about 20% of the time on Mongo 3.2 (I haven't tried 709 // other versions) with account balance being 3999 instead of 710 // 4000. That implies that some updates are being lost. This is 711 // bad and we'll need to chase it down in the near future - the 712 // only reason it's being skipped now is that it's already failing 713 // and it's better to have the txn tests running without this one 714 // than to have them not running at all. 715 if !*flaky { 716 c.Skip("Fails intermittently - disabling until fixed") 717 } 718 txn.SetChaos(txn.Chaos{ 719 SlowdownChance: 0.3, 720 Slowdown: 50 * time.Millisecond, 721 }) 722 defer txn.SetChaos(txn.Chaos{}) 723 724 // So we can run more iterations of the test in less time. 725 txn.SetDebug(false) 726 727 err := s.accounts.Insert(M{"_id": 0, "balance": 0}, M{"_id": 1, "balance": 0}) 728 c.Assert(err, IsNil) 729 730 // Run half of the operations changing account 0 and then 1, 731 // and the other half in the opposite order. 732 ops01 := []txn.Op{{ 733 C: "accounts", 734 Id: 0, 735 Update: M{"$inc": M{"balance": 1}}, 736 }, { 737 C: "accounts", 738 Id: 1, 739 Update: M{"$inc": M{"balance": 1}}, 740 }} 741 742 ops10 := []txn.Op{{ 743 C: "accounts", 744 Id: 1, 745 Update: M{"$inc": M{"balance": 1}}, 746 }, { 747 C: "accounts", 748 Id: 0, 749 Update: M{"$inc": M{"balance": 1}}, 750 }} 751 752 ops := [][]txn.Op{ops01, ops10} 753 754 const runners = 4 755 const changes = 1000 756 757 var wg sync.WaitGroup 758 wg.Add(runners) 759 for n := 0; n < runners; n++ { 760 n := n 761 go func() { 762 defer wg.Done() 763 for i := 0; i < changes; i++ { 764 err = s.runner.Run(ops[n%2], "", nil) 765 c.Assert(err, IsNil) 766 } 767 }() 768 } 769 wg.Wait() 770 771 for id := 0; id < 2; id++ { 772 var account Account 773 err = s.accounts.FindId(id).One(&account) 774 if account.Balance != runners*changes { 775 c.Errorf("Account should have balance of %d, got %d", runners*changes, account.Balance) 776 } 777 } 778} 779