1package bolt_test 2 3import ( 4 "bytes" 5 "encoding/binary" 6 "errors" 7 "flag" 8 "fmt" 9 "hash/fnv" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "sort" 16 "strings" 17 "sync" 18 "testing" 19 "time" 20 "unsafe" 21 22 "github.com/boltdb/bolt" 23) 24 25var statsFlag = flag.Bool("stats", false, "show performance stats") 26 27// version is the data file format version. 28const version = 2 29 30// magic is the marker value to indicate that a file is a Bolt DB. 31const magic uint32 = 0xED0CDAED 32 33// pageSize is the size of one page in the data file. 34const pageSize = 4096 35 36// pageHeaderSize is the size of a page header. 37const pageHeaderSize = 16 38 39// meta represents a simplified version of a database meta page for testing. 40type meta struct { 41 magic uint32 42 version uint32 43 _ uint32 44 _ uint32 45 _ [16]byte 46 _ uint64 47 pgid uint64 48 _ uint64 49 checksum uint64 50} 51 52// Ensure that a database can be opened without error. 53func TestOpen(t *testing.T) { 54 path := tempfile() 55 db, err := bolt.Open(path, 0666, nil) 56 if err != nil { 57 t.Fatal(err) 58 } else if db == nil { 59 t.Fatal("expected db") 60 } 61 62 if s := db.Path(); s != path { 63 t.Fatalf("unexpected path: %s", s) 64 } 65 66 if err := db.Close(); err != nil { 67 t.Fatal(err) 68 } 69} 70 71// Ensure that opening a database with a blank path returns an error. 72func TestOpen_ErrPathRequired(t *testing.T) { 73 _, err := bolt.Open("", 0666, nil) 74 if err == nil { 75 t.Fatalf("expected error") 76 } 77} 78 79// Ensure that opening a database with a bad path returns an error. 80func TestOpen_ErrNotExists(t *testing.T) { 81 _, err := bolt.Open(filepath.Join(tempfile(), "bad-path"), 0666, nil) 82 if err == nil { 83 t.Fatal("expected error") 84 } 85} 86 87// Ensure that opening a file that is not a Bolt database returns ErrInvalid. 88func TestOpen_ErrInvalid(t *testing.T) { 89 path := tempfile() 90 91 f, err := os.Create(path) 92 if err != nil { 93 t.Fatal(err) 94 } 95 if _, err := fmt.Fprintln(f, "this is not a bolt database"); err != nil { 96 t.Fatal(err) 97 } 98 if err := f.Close(); err != nil { 99 t.Fatal(err) 100 } 101 defer os.Remove(path) 102 103 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrInvalid { 104 t.Fatalf("unexpected error: %s", err) 105 } 106} 107 108// Ensure that opening a file with two invalid versions returns ErrVersionMismatch. 109func TestOpen_ErrVersionMismatch(t *testing.T) { 110 if pageSize != os.Getpagesize() { 111 t.Skip("page size mismatch") 112 } 113 114 // Create empty database. 115 db := MustOpenDB() 116 path := db.Path() 117 defer db.MustClose() 118 119 // Close database. 120 if err := db.DB.Close(); err != nil { 121 t.Fatal(err) 122 } 123 124 // Read data file. 125 buf, err := ioutil.ReadFile(path) 126 if err != nil { 127 t.Fatal(err) 128 } 129 130 // Rewrite meta pages. 131 meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize])) 132 meta0.version++ 133 meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize])) 134 meta1.version++ 135 if err := ioutil.WriteFile(path, buf, 0666); err != nil { 136 t.Fatal(err) 137 } 138 139 // Reopen data file. 140 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrVersionMismatch { 141 t.Fatalf("unexpected error: %s", err) 142 } 143} 144 145// Ensure that opening a file with two invalid checksums returns ErrChecksum. 146func TestOpen_ErrChecksum(t *testing.T) { 147 if pageSize != os.Getpagesize() { 148 t.Skip("page size mismatch") 149 } 150 151 // Create empty database. 152 db := MustOpenDB() 153 path := db.Path() 154 defer db.MustClose() 155 156 // Close database. 157 if err := db.DB.Close(); err != nil { 158 t.Fatal(err) 159 } 160 161 // Read data file. 162 buf, err := ioutil.ReadFile(path) 163 if err != nil { 164 t.Fatal(err) 165 } 166 167 // Rewrite meta pages. 168 meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize])) 169 meta0.pgid++ 170 meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize])) 171 meta1.pgid++ 172 if err := ioutil.WriteFile(path, buf, 0666); err != nil { 173 t.Fatal(err) 174 } 175 176 // Reopen data file. 177 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrChecksum { 178 t.Fatalf("unexpected error: %s", err) 179 } 180} 181 182// Ensure that opening a database does not increase its size. 183// https://github.com/boltdb/bolt/issues/291 184func TestOpen_Size(t *testing.T) { 185 // Open a data file. 186 db := MustOpenDB() 187 path := db.Path() 188 defer db.MustClose() 189 190 pagesize := db.Info().PageSize 191 192 // Insert until we get above the minimum 4MB size. 193 if err := db.Update(func(tx *bolt.Tx) error { 194 b, _ := tx.CreateBucketIfNotExists([]byte("data")) 195 for i := 0; i < 10000; i++ { 196 if err := b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000)); err != nil { 197 t.Fatal(err) 198 } 199 } 200 return nil 201 }); err != nil { 202 t.Fatal(err) 203 } 204 205 // Close database and grab the size. 206 if err := db.DB.Close(); err != nil { 207 t.Fatal(err) 208 } 209 sz := fileSize(path) 210 if sz == 0 { 211 t.Fatalf("unexpected new file size: %d", sz) 212 } 213 214 // Reopen database, update, and check size again. 215 db0, err := bolt.Open(path, 0666, nil) 216 if err != nil { 217 t.Fatal(err) 218 } 219 if err := db0.Update(func(tx *bolt.Tx) error { 220 if err := tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}); err != nil { 221 t.Fatal(err) 222 } 223 return nil 224 }); err != nil { 225 t.Fatal(err) 226 } 227 if err := db0.Close(); err != nil { 228 t.Fatal(err) 229 } 230 newSz := fileSize(path) 231 if newSz == 0 { 232 t.Fatalf("unexpected new file size: %d", newSz) 233 } 234 235 // Compare the original size with the new size. 236 // db size might increase by a few page sizes due to the new small update. 237 if sz < newSz-5*int64(pagesize) { 238 t.Fatalf("unexpected file growth: %d => %d", sz, newSz) 239 } 240} 241 242// Ensure that opening a database beyond the max step size does not increase its size. 243// https://github.com/boltdb/bolt/issues/303 244func TestOpen_Size_Large(t *testing.T) { 245 if testing.Short() { 246 t.Skip("short mode") 247 } 248 249 // Open a data file. 250 db := MustOpenDB() 251 path := db.Path() 252 defer db.MustClose() 253 254 pagesize := db.Info().PageSize 255 256 // Insert until we get above the minimum 4MB size. 257 var index uint64 258 for i := 0; i < 10000; i++ { 259 if err := db.Update(func(tx *bolt.Tx) error { 260 b, _ := tx.CreateBucketIfNotExists([]byte("data")) 261 for j := 0; j < 1000; j++ { 262 if err := b.Put(u64tob(index), make([]byte, 50)); err != nil { 263 t.Fatal(err) 264 } 265 index++ 266 } 267 return nil 268 }); err != nil { 269 t.Fatal(err) 270 } 271 } 272 273 // Close database and grab the size. 274 if err := db.DB.Close(); err != nil { 275 t.Fatal(err) 276 } 277 sz := fileSize(path) 278 if sz == 0 { 279 t.Fatalf("unexpected new file size: %d", sz) 280 } else if sz < (1 << 30) { 281 t.Fatalf("expected larger initial size: %d", sz) 282 } 283 284 // Reopen database, update, and check size again. 285 db0, err := bolt.Open(path, 0666, nil) 286 if err != nil { 287 t.Fatal(err) 288 } 289 if err := db0.Update(func(tx *bolt.Tx) error { 290 return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) 291 }); err != nil { 292 t.Fatal(err) 293 } 294 if err := db0.Close(); err != nil { 295 t.Fatal(err) 296 } 297 298 newSz := fileSize(path) 299 if newSz == 0 { 300 t.Fatalf("unexpected new file size: %d", newSz) 301 } 302 303 // Compare the original size with the new size. 304 // db size might increase by a few page sizes due to the new small update. 305 if sz < newSz-5*int64(pagesize) { 306 t.Fatalf("unexpected file growth: %d => %d", sz, newSz) 307 } 308} 309 310// Ensure that a re-opened database is consistent. 311func TestOpen_Check(t *testing.T) { 312 path := tempfile() 313 314 db, err := bolt.Open(path, 0666, nil) 315 if err != nil { 316 t.Fatal(err) 317 } 318 if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { 319 t.Fatal(err) 320 } 321 if err := db.Close(); err != nil { 322 t.Fatal(err) 323 } 324 325 db, err = bolt.Open(path, 0666, nil) 326 if err != nil { 327 t.Fatal(err) 328 } 329 if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { 330 t.Fatal(err) 331 } 332 if err := db.Close(); err != nil { 333 t.Fatal(err) 334 } 335} 336 337// Ensure that write errors to the meta file handler during initialization are returned. 338func TestOpen_MetaInitWriteError(t *testing.T) { 339 t.Skip("pending") 340} 341 342// Ensure that a database that is too small returns an error. 343func TestOpen_FileTooSmall(t *testing.T) { 344 path := tempfile() 345 346 db, err := bolt.Open(path, 0666, nil) 347 if err != nil { 348 t.Fatal(err) 349 } 350 if err := db.Close(); err != nil { 351 t.Fatal(err) 352 } 353 354 // corrupt the database 355 if err := os.Truncate(path, int64(os.Getpagesize())); err != nil { 356 t.Fatal(err) 357 } 358 359 db, err = bolt.Open(path, 0666, nil) 360 if err == nil || err.Error() != "file size too small" { 361 t.Fatalf("unexpected error: %s", err) 362 } 363} 364 365// TestDB_Open_InitialMmapSize tests if having InitialMmapSize large enough 366// to hold data from concurrent write transaction resolves the issue that 367// read transaction blocks the write transaction and causes deadlock. 368// This is a very hacky test since the mmap size is not exposed. 369func TestDB_Open_InitialMmapSize(t *testing.T) { 370 path := tempfile() 371 defer os.Remove(path) 372 373 initMmapSize := 1 << 31 // 2GB 374 testWriteSize := 1 << 27 // 134MB 375 376 db, err := bolt.Open(path, 0666, &bolt.Options{InitialMmapSize: initMmapSize}) 377 if err != nil { 378 t.Fatal(err) 379 } 380 381 // create a long-running read transaction 382 // that never gets closed while writing 383 rtx, err := db.Begin(false) 384 if err != nil { 385 t.Fatal(err) 386 } 387 388 // create a write transaction 389 wtx, err := db.Begin(true) 390 if err != nil { 391 t.Fatal(err) 392 } 393 394 b, err := wtx.CreateBucket([]byte("test")) 395 if err != nil { 396 t.Fatal(err) 397 } 398 399 // and commit a large write 400 err = b.Put([]byte("foo"), make([]byte, testWriteSize)) 401 if err != nil { 402 t.Fatal(err) 403 } 404 405 done := make(chan struct{}) 406 407 go func() { 408 if err := wtx.Commit(); err != nil { 409 t.Fatal(err) 410 } 411 done <- struct{}{} 412 }() 413 414 select { 415 case <-time.After(5 * time.Second): 416 t.Errorf("unexpected that the reader blocks writer") 417 case <-done: 418 } 419 420 if err := rtx.Rollback(); err != nil { 421 t.Fatal(err) 422 } 423} 424 425// Ensure that a database cannot open a transaction when it's not open. 426func TestDB_Begin_ErrDatabaseNotOpen(t *testing.T) { 427 var db bolt.DB 428 if _, err := db.Begin(false); err != bolt.ErrDatabaseNotOpen { 429 t.Fatalf("unexpected error: %s", err) 430 } 431} 432 433// Ensure that a read-write transaction can be retrieved. 434func TestDB_BeginRW(t *testing.T) { 435 db := MustOpenDB() 436 defer db.MustClose() 437 438 tx, err := db.Begin(true) 439 if err != nil { 440 t.Fatal(err) 441 } else if tx == nil { 442 t.Fatal("expected tx") 443 } 444 445 if tx.DB() != db.DB { 446 t.Fatal("unexpected tx database") 447 } else if !tx.Writable() { 448 t.Fatal("expected writable tx") 449 } 450 451 if err := tx.Commit(); err != nil { 452 t.Fatal(err) 453 } 454} 455 456// Ensure that opening a transaction while the DB is closed returns an error. 457func TestDB_BeginRW_Closed(t *testing.T) { 458 var db bolt.DB 459 if _, err := db.Begin(true); err != bolt.ErrDatabaseNotOpen { 460 t.Fatalf("unexpected error: %s", err) 461 } 462} 463 464func TestDB_Close_PendingTx_RW(t *testing.T) { testDB_Close_PendingTx(t, true) } 465func TestDB_Close_PendingTx_RO(t *testing.T) { testDB_Close_PendingTx(t, false) } 466 467// Ensure that a database cannot close while transactions are open. 468func testDB_Close_PendingTx(t *testing.T, writable bool) { 469 db := MustOpenDB() 470 defer db.MustClose() 471 472 // Start transaction. 473 tx, err := db.Begin(true) 474 if err != nil { 475 t.Fatal(err) 476 } 477 478 // Open update in separate goroutine. 479 done := make(chan struct{}) 480 go func() { 481 if err := db.Close(); err != nil { 482 t.Fatal(err) 483 } 484 close(done) 485 }() 486 487 // Ensure database hasn't closed. 488 time.Sleep(100 * time.Millisecond) 489 select { 490 case <-done: 491 t.Fatal("database closed too early") 492 default: 493 } 494 495 // Commit transaction. 496 if err := tx.Commit(); err != nil { 497 t.Fatal(err) 498 } 499 500 // Ensure database closed now. 501 time.Sleep(100 * time.Millisecond) 502 select { 503 case <-done: 504 default: 505 t.Fatal("database did not close") 506 } 507} 508 509// Ensure a database can provide a transactional block. 510func TestDB_Update(t *testing.T) { 511 db := MustOpenDB() 512 defer db.MustClose() 513 if err := db.Update(func(tx *bolt.Tx) error { 514 b, err := tx.CreateBucket([]byte("widgets")) 515 if err != nil { 516 t.Fatal(err) 517 } 518 if err := b.Put([]byte("foo"), []byte("bar")); err != nil { 519 t.Fatal(err) 520 } 521 if err := b.Put([]byte("baz"), []byte("bat")); err != nil { 522 t.Fatal(err) 523 } 524 if err := b.Delete([]byte("foo")); err != nil { 525 t.Fatal(err) 526 } 527 return nil 528 }); err != nil { 529 t.Fatal(err) 530 } 531 if err := db.View(func(tx *bolt.Tx) error { 532 b := tx.Bucket([]byte("widgets")) 533 if v := b.Get([]byte("foo")); v != nil { 534 t.Fatalf("expected nil value, got: %v", v) 535 } 536 if v := b.Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { 537 t.Fatalf("unexpected value: %v", v) 538 } 539 return nil 540 }); err != nil { 541 t.Fatal(err) 542 } 543} 544 545// Ensure a closed database returns an error while running a transaction block 546func TestDB_Update_Closed(t *testing.T) { 547 var db bolt.DB 548 if err := db.Update(func(tx *bolt.Tx) error { 549 if _, err := tx.CreateBucket([]byte("widgets")); err != nil { 550 t.Fatal(err) 551 } 552 return nil 553 }); err != bolt.ErrDatabaseNotOpen { 554 t.Fatalf("unexpected error: %s", err) 555 } 556} 557 558// Ensure a panic occurs while trying to commit a managed transaction. 559func TestDB_Update_ManualCommit(t *testing.T) { 560 db := MustOpenDB() 561 defer db.MustClose() 562 563 var panicked bool 564 if err := db.Update(func(tx *bolt.Tx) error { 565 func() { 566 defer func() { 567 if r := recover(); r != nil { 568 panicked = true 569 } 570 }() 571 572 if err := tx.Commit(); err != nil { 573 t.Fatal(err) 574 } 575 }() 576 return nil 577 }); err != nil { 578 t.Fatal(err) 579 } else if !panicked { 580 t.Fatal("expected panic") 581 } 582} 583 584// Ensure a panic occurs while trying to rollback a managed transaction. 585func TestDB_Update_ManualRollback(t *testing.T) { 586 db := MustOpenDB() 587 defer db.MustClose() 588 589 var panicked bool 590 if err := db.Update(func(tx *bolt.Tx) error { 591 func() { 592 defer func() { 593 if r := recover(); r != nil { 594 panicked = true 595 } 596 }() 597 598 if err := tx.Rollback(); err != nil { 599 t.Fatal(err) 600 } 601 }() 602 return nil 603 }); err != nil { 604 t.Fatal(err) 605 } else if !panicked { 606 t.Fatal("expected panic") 607 } 608} 609 610// Ensure a panic occurs while trying to commit a managed transaction. 611func TestDB_View_ManualCommit(t *testing.T) { 612 db := MustOpenDB() 613 defer db.MustClose() 614 615 var panicked bool 616 if err := db.View(func(tx *bolt.Tx) error { 617 func() { 618 defer func() { 619 if r := recover(); r != nil { 620 panicked = true 621 } 622 }() 623 624 if err := tx.Commit(); err != nil { 625 t.Fatal(err) 626 } 627 }() 628 return nil 629 }); err != nil { 630 t.Fatal(err) 631 } else if !panicked { 632 t.Fatal("expected panic") 633 } 634} 635 636// Ensure a panic occurs while trying to rollback a managed transaction. 637func TestDB_View_ManualRollback(t *testing.T) { 638 db := MustOpenDB() 639 defer db.MustClose() 640 641 var panicked bool 642 if err := db.View(func(tx *bolt.Tx) error { 643 func() { 644 defer func() { 645 if r := recover(); r != nil { 646 panicked = true 647 } 648 }() 649 650 if err := tx.Rollback(); err != nil { 651 t.Fatal(err) 652 } 653 }() 654 return nil 655 }); err != nil { 656 t.Fatal(err) 657 } else if !panicked { 658 t.Fatal("expected panic") 659 } 660} 661 662// Ensure a write transaction that panics does not hold open locks. 663func TestDB_Update_Panic(t *testing.T) { 664 db := MustOpenDB() 665 defer db.MustClose() 666 667 // Panic during update but recover. 668 func() { 669 defer func() { 670 if r := recover(); r != nil { 671 t.Log("recover: update", r) 672 } 673 }() 674 675 if err := db.Update(func(tx *bolt.Tx) error { 676 if _, err := tx.CreateBucket([]byte("widgets")); err != nil { 677 t.Fatal(err) 678 } 679 panic("omg") 680 }); err != nil { 681 t.Fatal(err) 682 } 683 }() 684 685 // Verify we can update again. 686 if err := db.Update(func(tx *bolt.Tx) error { 687 if _, err := tx.CreateBucket([]byte("widgets")); err != nil { 688 t.Fatal(err) 689 } 690 return nil 691 }); err != nil { 692 t.Fatal(err) 693 } 694 695 // Verify that our change persisted. 696 if err := db.Update(func(tx *bolt.Tx) error { 697 if tx.Bucket([]byte("widgets")) == nil { 698 t.Fatal("expected bucket") 699 } 700 return nil 701 }); err != nil { 702 t.Fatal(err) 703 } 704} 705 706// Ensure a database can return an error through a read-only transactional block. 707func TestDB_View_Error(t *testing.T) { 708 db := MustOpenDB() 709 defer db.MustClose() 710 711 if err := db.View(func(tx *bolt.Tx) error { 712 return errors.New("xxx") 713 }); err == nil || err.Error() != "xxx" { 714 t.Fatalf("unexpected error: %s", err) 715 } 716} 717 718// Ensure a read transaction that panics does not hold open locks. 719func TestDB_View_Panic(t *testing.T) { 720 db := MustOpenDB() 721 defer db.MustClose() 722 723 if err := db.Update(func(tx *bolt.Tx) error { 724 if _, err := tx.CreateBucket([]byte("widgets")); err != nil { 725 t.Fatal(err) 726 } 727 return nil 728 }); err != nil { 729 t.Fatal(err) 730 } 731 732 // Panic during view transaction but recover. 733 func() { 734 defer func() { 735 if r := recover(); r != nil { 736 t.Log("recover: view", r) 737 } 738 }() 739 740 if err := db.View(func(tx *bolt.Tx) error { 741 if tx.Bucket([]byte("widgets")) == nil { 742 t.Fatal("expected bucket") 743 } 744 panic("omg") 745 }); err != nil { 746 t.Fatal(err) 747 } 748 }() 749 750 // Verify that we can still use read transactions. 751 if err := db.View(func(tx *bolt.Tx) error { 752 if tx.Bucket([]byte("widgets")) == nil { 753 t.Fatal("expected bucket") 754 } 755 return nil 756 }); err != nil { 757 t.Fatal(err) 758 } 759} 760 761// Ensure that DB stats can be returned. 762func TestDB_Stats(t *testing.T) { 763 db := MustOpenDB() 764 defer db.MustClose() 765 if err := db.Update(func(tx *bolt.Tx) error { 766 _, err := tx.CreateBucket([]byte("widgets")) 767 return err 768 }); err != nil { 769 t.Fatal(err) 770 } 771 772 stats := db.Stats() 773 if stats.TxStats.PageCount != 2 { 774 t.Fatalf("unexpected TxStats.PageCount: %d", stats.TxStats.PageCount) 775 } else if stats.FreePageN != 0 { 776 t.Fatalf("unexpected FreePageN != 0: %d", stats.FreePageN) 777 } else if stats.PendingPageN != 2 { 778 t.Fatalf("unexpected PendingPageN != 2: %d", stats.PendingPageN) 779 } 780} 781 782// Ensure that database pages are in expected order and type. 783func TestDB_Consistency(t *testing.T) { 784 db := MustOpenDB() 785 defer db.MustClose() 786 if err := db.Update(func(tx *bolt.Tx) error { 787 _, err := tx.CreateBucket([]byte("widgets")) 788 return err 789 }); err != nil { 790 t.Fatal(err) 791 } 792 793 for i := 0; i < 10; i++ { 794 if err := db.Update(func(tx *bolt.Tx) error { 795 if err := tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")); err != nil { 796 t.Fatal(err) 797 } 798 return nil 799 }); err != nil { 800 t.Fatal(err) 801 } 802 } 803 804 if err := db.Update(func(tx *bolt.Tx) error { 805 if p, _ := tx.Page(0); p == nil { 806 t.Fatal("expected page") 807 } else if p.Type != "meta" { 808 t.Fatalf("unexpected page type: %s", p.Type) 809 } 810 811 if p, _ := tx.Page(1); p == nil { 812 t.Fatal("expected page") 813 } else if p.Type != "meta" { 814 t.Fatalf("unexpected page type: %s", p.Type) 815 } 816 817 if p, _ := tx.Page(2); p == nil { 818 t.Fatal("expected page") 819 } else if p.Type != "free" { 820 t.Fatalf("unexpected page type: %s", p.Type) 821 } 822 823 if p, _ := tx.Page(3); p == nil { 824 t.Fatal("expected page") 825 } else if p.Type != "free" { 826 t.Fatalf("unexpected page type: %s", p.Type) 827 } 828 829 if p, _ := tx.Page(4); p == nil { 830 t.Fatal("expected page") 831 } else if p.Type != "leaf" { 832 t.Fatalf("unexpected page type: %s", p.Type) 833 } 834 835 if p, _ := tx.Page(5); p == nil { 836 t.Fatal("expected page") 837 } else if p.Type != "freelist" { 838 t.Fatalf("unexpected page type: %s", p.Type) 839 } 840 841 if p, _ := tx.Page(6); p != nil { 842 t.Fatal("unexpected page") 843 } 844 return nil 845 }); err != nil { 846 t.Fatal(err) 847 } 848} 849 850// Ensure that DB stats can be subtracted from one another. 851func TestDBStats_Sub(t *testing.T) { 852 var a, b bolt.Stats 853 a.TxStats.PageCount = 3 854 a.FreePageN = 4 855 b.TxStats.PageCount = 10 856 b.FreePageN = 14 857 diff := b.Sub(&a) 858 if diff.TxStats.PageCount != 7 { 859 t.Fatalf("unexpected TxStats.PageCount: %d", diff.TxStats.PageCount) 860 } 861 862 // free page stats are copied from the receiver and not subtracted 863 if diff.FreePageN != 14 { 864 t.Fatalf("unexpected FreePageN: %d", diff.FreePageN) 865 } 866} 867 868// Ensure two functions can perform updates in a single batch. 869func TestDB_Batch(t *testing.T) { 870 db := MustOpenDB() 871 defer db.MustClose() 872 873 if err := db.Update(func(tx *bolt.Tx) error { 874 if _, err := tx.CreateBucket([]byte("widgets")); err != nil { 875 t.Fatal(err) 876 } 877 return nil 878 }); err != nil { 879 t.Fatal(err) 880 } 881 882 // Iterate over multiple updates in separate goroutines. 883 n := 2 884 ch := make(chan error) 885 for i := 0; i < n; i++ { 886 go func(i int) { 887 ch <- db.Batch(func(tx *bolt.Tx) error { 888 return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) 889 }) 890 }(i) 891 } 892 893 // Check all responses to make sure there's no error. 894 for i := 0; i < n; i++ { 895 if err := <-ch; err != nil { 896 t.Fatal(err) 897 } 898 } 899 900 // Ensure data is correct. 901 if err := db.View(func(tx *bolt.Tx) error { 902 b := tx.Bucket([]byte("widgets")) 903 for i := 0; i < n; i++ { 904 if v := b.Get(u64tob(uint64(i))); v == nil { 905 t.Errorf("key not found: %d", i) 906 } 907 } 908 return nil 909 }); err != nil { 910 t.Fatal(err) 911 } 912} 913 914func TestDB_Batch_Panic(t *testing.T) { 915 db := MustOpenDB() 916 defer db.MustClose() 917 918 var sentinel int 919 var bork = &sentinel 920 var problem interface{} 921 var err error 922 923 // Execute a function inside a batch that panics. 924 func() { 925 defer func() { 926 if p := recover(); p != nil { 927 problem = p 928 } 929 }() 930 err = db.Batch(func(tx *bolt.Tx) error { 931 panic(bork) 932 }) 933 }() 934 935 // Verify there is no error. 936 if g, e := err, error(nil); g != e { 937 t.Fatalf("wrong error: %v != %v", g, e) 938 } 939 // Verify the panic was captured. 940 if g, e := problem, bork; g != e { 941 t.Fatalf("wrong error: %v != %v", g, e) 942 } 943} 944 945func TestDB_BatchFull(t *testing.T) { 946 db := MustOpenDB() 947 defer db.MustClose() 948 if err := db.Update(func(tx *bolt.Tx) error { 949 _, err := tx.CreateBucket([]byte("widgets")) 950 return err 951 }); err != nil { 952 t.Fatal(err) 953 } 954 955 const size = 3 956 // buffered so we never leak goroutines 957 ch := make(chan error, size) 958 put := func(i int) { 959 ch <- db.Batch(func(tx *bolt.Tx) error { 960 return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) 961 }) 962 } 963 964 db.MaxBatchSize = size 965 // high enough to never trigger here 966 db.MaxBatchDelay = 1 * time.Hour 967 968 go put(1) 969 go put(2) 970 971 // Give the batch a chance to exhibit bugs. 972 time.Sleep(10 * time.Millisecond) 973 974 // not triggered yet 975 select { 976 case <-ch: 977 t.Fatalf("batch triggered too early") 978 default: 979 } 980 981 go put(3) 982 983 // Check all responses to make sure there's no error. 984 for i := 0; i < size; i++ { 985 if err := <-ch; err != nil { 986 t.Fatal(err) 987 } 988 } 989 990 // Ensure data is correct. 991 if err := db.View(func(tx *bolt.Tx) error { 992 b := tx.Bucket([]byte("widgets")) 993 for i := 1; i <= size; i++ { 994 if v := b.Get(u64tob(uint64(i))); v == nil { 995 t.Errorf("key not found: %d", i) 996 } 997 } 998 return nil 999 }); err != nil { 1000 t.Fatal(err) 1001 } 1002} 1003 1004func TestDB_BatchTime(t *testing.T) { 1005 db := MustOpenDB() 1006 defer db.MustClose() 1007 if err := db.Update(func(tx *bolt.Tx) error { 1008 _, err := tx.CreateBucket([]byte("widgets")) 1009 return err 1010 }); err != nil { 1011 t.Fatal(err) 1012 } 1013 1014 const size = 1 1015 // buffered so we never leak goroutines 1016 ch := make(chan error, size) 1017 put := func(i int) { 1018 ch <- db.Batch(func(tx *bolt.Tx) error { 1019 return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) 1020 }) 1021 } 1022 1023 db.MaxBatchSize = 1000 1024 db.MaxBatchDelay = 0 1025 1026 go put(1) 1027 1028 // Batch must trigger by time alone. 1029 1030 // Check all responses to make sure there's no error. 1031 for i := 0; i < size; i++ { 1032 if err := <-ch; err != nil { 1033 t.Fatal(err) 1034 } 1035 } 1036 1037 // Ensure data is correct. 1038 if err := db.View(func(tx *bolt.Tx) error { 1039 b := tx.Bucket([]byte("widgets")) 1040 for i := 1; i <= size; i++ { 1041 if v := b.Get(u64tob(uint64(i))); v == nil { 1042 t.Errorf("key not found: %d", i) 1043 } 1044 } 1045 return nil 1046 }); err != nil { 1047 t.Fatal(err) 1048 } 1049} 1050 1051func ExampleDB_Update() { 1052 // Open the database. 1053 db, err := bolt.Open(tempfile(), 0666, nil) 1054 if err != nil { 1055 log.Fatal(err) 1056 } 1057 defer os.Remove(db.Path()) 1058 1059 // Execute several commands within a read-write transaction. 1060 if err := db.Update(func(tx *bolt.Tx) error { 1061 b, err := tx.CreateBucket([]byte("widgets")) 1062 if err != nil { 1063 return err 1064 } 1065 if err := b.Put([]byte("foo"), []byte("bar")); err != nil { 1066 return err 1067 } 1068 return nil 1069 }); err != nil { 1070 log.Fatal(err) 1071 } 1072 1073 // Read the value back from a separate read-only transaction. 1074 if err := db.View(func(tx *bolt.Tx) error { 1075 value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) 1076 fmt.Printf("The value of 'foo' is: %s\n", value) 1077 return nil 1078 }); err != nil { 1079 log.Fatal(err) 1080 } 1081 1082 // Close database to release the file lock. 1083 if err := db.Close(); err != nil { 1084 log.Fatal(err) 1085 } 1086 1087 // Output: 1088 // The value of 'foo' is: bar 1089} 1090 1091func ExampleDB_View() { 1092 // Open the database. 1093 db, err := bolt.Open(tempfile(), 0666, nil) 1094 if err != nil { 1095 log.Fatal(err) 1096 } 1097 defer os.Remove(db.Path()) 1098 1099 // Insert data into a bucket. 1100 if err := db.Update(func(tx *bolt.Tx) error { 1101 b, err := tx.CreateBucket([]byte("people")) 1102 if err != nil { 1103 return err 1104 } 1105 if err := b.Put([]byte("john"), []byte("doe")); err != nil { 1106 return err 1107 } 1108 if err := b.Put([]byte("susy"), []byte("que")); err != nil { 1109 return err 1110 } 1111 return nil 1112 }); err != nil { 1113 log.Fatal(err) 1114 } 1115 1116 // Access data from within a read-only transactional block. 1117 if err := db.View(func(tx *bolt.Tx) error { 1118 v := tx.Bucket([]byte("people")).Get([]byte("john")) 1119 fmt.Printf("John's last name is %s.\n", v) 1120 return nil 1121 }); err != nil { 1122 log.Fatal(err) 1123 } 1124 1125 // Close database to release the file lock. 1126 if err := db.Close(); err != nil { 1127 log.Fatal(err) 1128 } 1129 1130 // Output: 1131 // John's last name is doe. 1132} 1133 1134func ExampleDB_Begin_ReadOnly() { 1135 // Open the database. 1136 db, err := bolt.Open(tempfile(), 0666, nil) 1137 if err != nil { 1138 log.Fatal(err) 1139 } 1140 defer os.Remove(db.Path()) 1141 1142 // Create a bucket using a read-write transaction. 1143 if err := db.Update(func(tx *bolt.Tx) error { 1144 _, err := tx.CreateBucket([]byte("widgets")) 1145 return err 1146 }); err != nil { 1147 log.Fatal(err) 1148 } 1149 1150 // Create several keys in a transaction. 1151 tx, err := db.Begin(true) 1152 if err != nil { 1153 log.Fatal(err) 1154 } 1155 b := tx.Bucket([]byte("widgets")) 1156 if err := b.Put([]byte("john"), []byte("blue")); err != nil { 1157 log.Fatal(err) 1158 } 1159 if err := b.Put([]byte("abby"), []byte("red")); err != nil { 1160 log.Fatal(err) 1161 } 1162 if err := b.Put([]byte("zephyr"), []byte("purple")); err != nil { 1163 log.Fatal(err) 1164 } 1165 if err := tx.Commit(); err != nil { 1166 log.Fatal(err) 1167 } 1168 1169 // Iterate over the values in sorted key order. 1170 tx, err = db.Begin(false) 1171 if err != nil { 1172 log.Fatal(err) 1173 } 1174 c := tx.Bucket([]byte("widgets")).Cursor() 1175 for k, v := c.First(); k != nil; k, v = c.Next() { 1176 fmt.Printf("%s likes %s\n", k, v) 1177 } 1178 1179 if err := tx.Rollback(); err != nil { 1180 log.Fatal(err) 1181 } 1182 1183 if err := db.Close(); err != nil { 1184 log.Fatal(err) 1185 } 1186 1187 // Output: 1188 // abby likes red 1189 // john likes blue 1190 // zephyr likes purple 1191} 1192 1193func BenchmarkDBBatchAutomatic(b *testing.B) { 1194 db := MustOpenDB() 1195 defer db.MustClose() 1196 if err := db.Update(func(tx *bolt.Tx) error { 1197 _, err := tx.CreateBucket([]byte("bench")) 1198 return err 1199 }); err != nil { 1200 b.Fatal(err) 1201 } 1202 1203 b.ResetTimer() 1204 for i := 0; i < b.N; i++ { 1205 start := make(chan struct{}) 1206 var wg sync.WaitGroup 1207 1208 for round := 0; round < 1000; round++ { 1209 wg.Add(1) 1210 1211 go func(id uint32) { 1212 defer wg.Done() 1213 <-start 1214 1215 h := fnv.New32a() 1216 buf := make([]byte, 4) 1217 binary.LittleEndian.PutUint32(buf, id) 1218 _, _ = h.Write(buf[:]) 1219 k := h.Sum(nil) 1220 insert := func(tx *bolt.Tx) error { 1221 b := tx.Bucket([]byte("bench")) 1222 return b.Put(k, []byte("filler")) 1223 } 1224 if err := db.Batch(insert); err != nil { 1225 b.Error(err) 1226 return 1227 } 1228 }(uint32(round)) 1229 } 1230 close(start) 1231 wg.Wait() 1232 } 1233 1234 b.StopTimer() 1235 validateBatchBench(b, db) 1236} 1237 1238func BenchmarkDBBatchSingle(b *testing.B) { 1239 db := MustOpenDB() 1240 defer db.MustClose() 1241 if err := db.Update(func(tx *bolt.Tx) error { 1242 _, err := tx.CreateBucket([]byte("bench")) 1243 return err 1244 }); err != nil { 1245 b.Fatal(err) 1246 } 1247 1248 b.ResetTimer() 1249 for i := 0; i < b.N; i++ { 1250 start := make(chan struct{}) 1251 var wg sync.WaitGroup 1252 1253 for round := 0; round < 1000; round++ { 1254 wg.Add(1) 1255 go func(id uint32) { 1256 defer wg.Done() 1257 <-start 1258 1259 h := fnv.New32a() 1260 buf := make([]byte, 4) 1261 binary.LittleEndian.PutUint32(buf, id) 1262 _, _ = h.Write(buf[:]) 1263 k := h.Sum(nil) 1264 insert := func(tx *bolt.Tx) error { 1265 b := tx.Bucket([]byte("bench")) 1266 return b.Put(k, []byte("filler")) 1267 } 1268 if err := db.Update(insert); err != nil { 1269 b.Error(err) 1270 return 1271 } 1272 }(uint32(round)) 1273 } 1274 close(start) 1275 wg.Wait() 1276 } 1277 1278 b.StopTimer() 1279 validateBatchBench(b, db) 1280} 1281 1282func BenchmarkDBBatchManual10x100(b *testing.B) { 1283 db := MustOpenDB() 1284 defer db.MustClose() 1285 if err := db.Update(func(tx *bolt.Tx) error { 1286 _, err := tx.CreateBucket([]byte("bench")) 1287 return err 1288 }); err != nil { 1289 b.Fatal(err) 1290 } 1291 1292 b.ResetTimer() 1293 for i := 0; i < b.N; i++ { 1294 start := make(chan struct{}) 1295 var wg sync.WaitGroup 1296 1297 for major := 0; major < 10; major++ { 1298 wg.Add(1) 1299 go func(id uint32) { 1300 defer wg.Done() 1301 <-start 1302 1303 insert100 := func(tx *bolt.Tx) error { 1304 h := fnv.New32a() 1305 buf := make([]byte, 4) 1306 for minor := uint32(0); minor < 100; minor++ { 1307 binary.LittleEndian.PutUint32(buf, uint32(id*100+minor)) 1308 h.Reset() 1309 _, _ = h.Write(buf[:]) 1310 k := h.Sum(nil) 1311 b := tx.Bucket([]byte("bench")) 1312 if err := b.Put(k, []byte("filler")); err != nil { 1313 return err 1314 } 1315 } 1316 return nil 1317 } 1318 if err := db.Update(insert100); err != nil { 1319 b.Fatal(err) 1320 } 1321 }(uint32(major)) 1322 } 1323 close(start) 1324 wg.Wait() 1325 } 1326 1327 b.StopTimer() 1328 validateBatchBench(b, db) 1329} 1330 1331func validateBatchBench(b *testing.B, db *DB) { 1332 var rollback = errors.New("sentinel error to cause rollback") 1333 validate := func(tx *bolt.Tx) error { 1334 bucket := tx.Bucket([]byte("bench")) 1335 h := fnv.New32a() 1336 buf := make([]byte, 4) 1337 for id := uint32(0); id < 1000; id++ { 1338 binary.LittleEndian.PutUint32(buf, id) 1339 h.Reset() 1340 _, _ = h.Write(buf[:]) 1341 k := h.Sum(nil) 1342 v := bucket.Get(k) 1343 if v == nil { 1344 b.Errorf("not found id=%d key=%x", id, k) 1345 continue 1346 } 1347 if g, e := v, []byte("filler"); !bytes.Equal(g, e) { 1348 b.Errorf("bad value for id=%d key=%x: %s != %q", id, k, g, e) 1349 } 1350 if err := bucket.Delete(k); err != nil { 1351 return err 1352 } 1353 } 1354 // should be empty now 1355 c := bucket.Cursor() 1356 for k, v := c.First(); k != nil; k, v = c.Next() { 1357 b.Errorf("unexpected key: %x = %q", k, v) 1358 } 1359 return rollback 1360 } 1361 if err := db.Update(validate); err != nil && err != rollback { 1362 b.Error(err) 1363 } 1364} 1365 1366// DB is a test wrapper for bolt.DB. 1367type DB struct { 1368 *bolt.DB 1369} 1370 1371// MustOpenDB returns a new, open DB at a temporary location. 1372func MustOpenDB() *DB { 1373 db, err := bolt.Open(tempfile(), 0666, nil) 1374 if err != nil { 1375 panic(err) 1376 } 1377 return &DB{db} 1378} 1379 1380// Close closes the database and deletes the underlying file. 1381func (db *DB) Close() error { 1382 // Log statistics. 1383 if *statsFlag { 1384 db.PrintStats() 1385 } 1386 1387 // Check database consistency after every test. 1388 db.MustCheck() 1389 1390 // Close database and remove file. 1391 defer os.Remove(db.Path()) 1392 return db.DB.Close() 1393} 1394 1395// MustClose closes the database and deletes the underlying file. Panic on error. 1396func (db *DB) MustClose() { 1397 if err := db.Close(); err != nil { 1398 panic(err) 1399 } 1400} 1401 1402// PrintStats prints the database stats 1403func (db *DB) PrintStats() { 1404 var stats = db.Stats() 1405 fmt.Printf("[db] %-20s %-20s %-20s\n", 1406 fmt.Sprintf("pg(%d/%d)", stats.TxStats.PageCount, stats.TxStats.PageAlloc), 1407 fmt.Sprintf("cur(%d)", stats.TxStats.CursorCount), 1408 fmt.Sprintf("node(%d/%d)", stats.TxStats.NodeCount, stats.TxStats.NodeDeref), 1409 ) 1410 fmt.Printf(" %-20s %-20s %-20s\n", 1411 fmt.Sprintf("rebal(%d/%v)", stats.TxStats.Rebalance, truncDuration(stats.TxStats.RebalanceTime)), 1412 fmt.Sprintf("spill(%d/%v)", stats.TxStats.Spill, truncDuration(stats.TxStats.SpillTime)), 1413 fmt.Sprintf("w(%d/%v)", stats.TxStats.Write, truncDuration(stats.TxStats.WriteTime)), 1414 ) 1415} 1416 1417// MustCheck runs a consistency check on the database and panics if any errors are found. 1418func (db *DB) MustCheck() { 1419 if err := db.Update(func(tx *bolt.Tx) error { 1420 // Collect all the errors. 1421 var errors []error 1422 for err := range tx.Check() { 1423 errors = append(errors, err) 1424 if len(errors) > 10 { 1425 break 1426 } 1427 } 1428 1429 // If errors occurred, copy the DB and print the errors. 1430 if len(errors) > 0 { 1431 var path = tempfile() 1432 if err := tx.CopyFile(path, 0600); err != nil { 1433 panic(err) 1434 } 1435 1436 // Print errors. 1437 fmt.Print("\n\n") 1438 fmt.Printf("consistency check failed (%d errors)\n", len(errors)) 1439 for _, err := range errors { 1440 fmt.Println(err) 1441 } 1442 fmt.Println("") 1443 fmt.Println("db saved to:") 1444 fmt.Println(path) 1445 fmt.Print("\n\n") 1446 os.Exit(-1) 1447 } 1448 1449 return nil 1450 }); err != nil && err != bolt.ErrDatabaseNotOpen { 1451 panic(err) 1452 } 1453} 1454 1455// CopyTempFile copies a database to a temporary file. 1456func (db *DB) CopyTempFile() { 1457 path := tempfile() 1458 if err := db.View(func(tx *bolt.Tx) error { 1459 return tx.CopyFile(path, 0600) 1460 }); err != nil { 1461 panic(err) 1462 } 1463 fmt.Println("db copied to: ", path) 1464} 1465 1466// tempfile returns a temporary file path. 1467func tempfile() string { 1468 f, err := ioutil.TempFile("", "bolt-") 1469 if err != nil { 1470 panic(err) 1471 } 1472 if err := f.Close(); err != nil { 1473 panic(err) 1474 } 1475 if err := os.Remove(f.Name()); err != nil { 1476 panic(err) 1477 } 1478 return f.Name() 1479} 1480 1481// mustContainKeys checks that a bucket contains a given set of keys. 1482func mustContainKeys(b *bolt.Bucket, m map[string]string) { 1483 found := make(map[string]string) 1484 if err := b.ForEach(func(k, _ []byte) error { 1485 found[string(k)] = "" 1486 return nil 1487 }); err != nil { 1488 panic(err) 1489 } 1490 1491 // Check for keys found in bucket that shouldn't be there. 1492 var keys []string 1493 for k, _ := range found { 1494 if _, ok := m[string(k)]; !ok { 1495 keys = append(keys, k) 1496 } 1497 } 1498 if len(keys) > 0 { 1499 sort.Strings(keys) 1500 panic(fmt.Sprintf("keys found(%d): %s", len(keys), strings.Join(keys, ","))) 1501 } 1502 1503 // Check for keys not found in bucket that should be there. 1504 for k, _ := range m { 1505 if _, ok := found[string(k)]; !ok { 1506 keys = append(keys, k) 1507 } 1508 } 1509 if len(keys) > 0 { 1510 sort.Strings(keys) 1511 panic(fmt.Sprintf("keys not found(%d): %s", len(keys), strings.Join(keys, ","))) 1512 } 1513} 1514 1515func trunc(b []byte, length int) []byte { 1516 if length < len(b) { 1517 return b[:length] 1518 } 1519 return b 1520} 1521 1522func truncDuration(d time.Duration) string { 1523 return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") 1524} 1525 1526func fileSize(path string) int64 { 1527 fi, err := os.Stat(path) 1528 if err != nil { 1529 return 0 1530 } 1531 return fi.Size() 1532} 1533 1534func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } 1535func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } 1536 1537// u64tob converts a uint64 into an 8-byte slice. 1538func u64tob(v uint64) []byte { 1539 b := make([]byte, 8) 1540 binary.BigEndian.PutUint64(b, v) 1541 return b 1542} 1543 1544// btou64 converts an 8-byte slice into an uint64. 1545func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } 1546