1package raft 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "reflect" 10 "sync" 11 "testing" 12 "time" 13 14 "github.com/hashicorp/go-hclog" 15 "github.com/hashicorp/go-msgpack/codec" 16) 17 18var ( 19 userSnapshotErrorsOnNoData = true 20) 21 22// Return configurations optimized for in-memory 23func inmemConfig(t *testing.T) *Config { 24 conf := DefaultConfig() 25 conf.HeartbeatTimeout = 50 * time.Millisecond 26 conf.ElectionTimeout = 50 * time.Millisecond 27 conf.LeaderLeaseTimeout = 50 * time.Millisecond 28 conf.CommitTimeout = 5 * time.Millisecond 29 conf.Logger = newTestLeveledLogger(t) 30 return conf 31} 32 33// MockFSM is an implementation of the FSM interface, and just stores 34// the logs sequentially. 35// 36// NOTE: This is exposed for middleware testing purposes and is not a stable API 37type MockFSM struct { 38 sync.Mutex 39 logs [][]byte 40 configurations []Configuration 41} 42 43// NOTE: This is exposed for middleware testing purposes and is not a stable API 44type MockFSMConfigStore struct { 45 FSM 46} 47 48// NOTE: This is exposed for middleware testing purposes and is not a stable API 49type WrappingFSM interface { 50 Underlying() FSM 51} 52 53func getMockFSM(fsm FSM) *MockFSM { 54 switch f := fsm.(type) { 55 case *MockFSM: 56 return f 57 case *MockFSMConfigStore: 58 return f.FSM.(*MockFSM) 59 case WrappingFSM: 60 return getMockFSM(f.Underlying()) 61 } 62 63 return nil 64} 65 66// NOTE: This is exposed for middleware testing purposes and is not a stable API 67type MockSnapshot struct { 68 logs [][]byte 69 maxIndex int 70} 71 72var _ ConfigurationStore = (*MockFSMConfigStore)(nil) 73 74// NOTE: This is exposed for middleware testing purposes and is not a stable API 75func (m *MockFSM) Apply(log *Log) interface{} { 76 m.Lock() 77 defer m.Unlock() 78 m.logs = append(m.logs, log.Data) 79 return len(m.logs) 80} 81 82// NOTE: This is exposed for middleware testing purposes and is not a stable API 83func (m *MockFSM) Snapshot() (FSMSnapshot, error) { 84 m.Lock() 85 defer m.Unlock() 86 return &MockSnapshot{m.logs, len(m.logs)}, nil 87} 88 89// NOTE: This is exposed for middleware testing purposes and is not a stable API 90func (m *MockFSM) Restore(inp io.ReadCloser) error { 91 m.Lock() 92 defer m.Unlock() 93 defer inp.Close() 94 hd := codec.MsgpackHandle{} 95 dec := codec.NewDecoder(inp, &hd) 96 97 m.logs = nil 98 return dec.Decode(&m.logs) 99} 100 101// NOTE: This is exposed for middleware testing purposes and is not a stable API 102func (m *MockFSM) Logs() [][]byte { 103 m.Lock() 104 defer m.Unlock() 105 return m.logs 106} 107 108// NOTE: This is exposed for middleware testing purposes and is not a stable API 109func (m *MockFSMConfigStore) StoreConfiguration(index uint64, config Configuration) { 110 mm := m.FSM.(*MockFSM) 111 mm.Lock() 112 defer mm.Unlock() 113 mm.configurations = append(mm.configurations, config) 114} 115 116// NOTE: This is exposed for middleware testing purposes and is not a stable API 117func (m *MockSnapshot) Persist(sink SnapshotSink) error { 118 hd := codec.MsgpackHandle{} 119 enc := codec.NewEncoder(sink, &hd) 120 if err := enc.Encode(m.logs[:m.maxIndex]); err != nil { 121 sink.Cancel() 122 return err 123 } 124 sink.Close() 125 return nil 126} 127 128// NOTE: This is exposed for middleware testing purposes and is not a stable API 129func (m *MockSnapshot) Release() { 130} 131 132// This can be used as the destination for a logger and it'll 133// map them into calls to testing.T.Log, so that you only see 134// the logging for failed tests. 135type testLoggerAdapter struct { 136 t *testing.T 137 prefix string 138} 139 140func (a *testLoggerAdapter) Write(d []byte) (int, error) { 141 if d[len(d)-1] == '\n' { 142 d = d[:len(d)-1] 143 } 144 if a.prefix != "" { 145 l := a.prefix + ": " + string(d) 146 if testing.Verbose() { 147 fmt.Printf("testLoggerAdapter verbose: %s\n", l) 148 } 149 a.t.Log(l) 150 return len(l), nil 151 } 152 153 a.t.Log(string(d)) 154 return len(d), nil 155} 156 157func newTestLogger(t *testing.T) hclog.Logger { 158 return hclog.New(&hclog.LoggerOptions{ 159 Output: &testLoggerAdapter{t: t}, 160 Level: hclog.DefaultLevel, 161 }) 162} 163 164func newTestLoggerWithPrefix(t *testing.T, prefix string) hclog.Logger { 165 return hclog.New(&hclog.LoggerOptions{ 166 Output: &testLoggerAdapter{t: t, prefix: prefix}, 167 Level: hclog.DefaultLevel, 168 }) 169} 170 171func newTestLeveledLogger(t *testing.T) hclog.Logger { 172 return hclog.New(&hclog.LoggerOptions{ 173 Name: "", 174 Output: &testLoggerAdapter{t: t}, 175 }) 176} 177 178func newTestLeveledLoggerWithPrefix(t *testing.T, prefix string) hclog.Logger { 179 return hclog.New(&hclog.LoggerOptions{ 180 Name: prefix, 181 Output: &testLoggerAdapter{t: t, prefix: prefix}, 182 }) 183} 184 185type cluster struct { 186 dirs []string 187 stores []*InmemStore 188 fsms []FSM 189 snaps []*FileSnapshotStore 190 trans []LoopbackTransport 191 rafts []*Raft 192 t *testing.T 193 observationCh chan Observation 194 conf *Config 195 propagateTimeout time.Duration 196 longstopTimeout time.Duration 197 logger hclog.Logger 198 startTime time.Time 199 200 failedLock sync.Mutex 201 failedCh chan struct{} 202 failed bool 203} 204 205func (c *cluster) Merge(other *cluster) { 206 c.dirs = append(c.dirs, other.dirs...) 207 c.stores = append(c.stores, other.stores...) 208 c.fsms = append(c.fsms, other.fsms...) 209 c.snaps = append(c.snaps, other.snaps...) 210 c.trans = append(c.trans, other.trans...) 211 c.rafts = append(c.rafts, other.rafts...) 212} 213 214// notifyFailed will close the failed channel which can signal the goroutine 215// running the test that another goroutine has detected a failure in order to 216// terminate the test. 217func (c *cluster) notifyFailed() { 218 c.failedLock.Lock() 219 defer c.failedLock.Unlock() 220 if !c.failed { 221 c.failed = true 222 close(c.failedCh) 223 } 224} 225 226// Failf provides a logging function that fails the tests, prints the output 227// with microseconds, and does not mysteriously eat the string. This can be 228// safely called from goroutines but won't immediately halt the test. The 229// failedCh will be closed to allow blocking functions in the main thread to 230// detect the failure and react. Note that you should arrange for the main 231// thread to block until all goroutines have completed in order to reliably 232// fail tests using this function. 233func (c *cluster) Failf(format string, args ...interface{}) { 234 c.logger.Error(fmt.Sprintf(format, args...)) 235 c.t.Fail() 236 c.notifyFailed() 237} 238 239// FailNowf provides a logging function that fails the tests, prints the output 240// with microseconds, and does not mysteriously eat the string. FailNowf must be 241// called from the goroutine running the test or benchmark function, not from 242// other goroutines created during the test. Calling FailNowf does not stop 243// those other goroutines. 244func (c *cluster) FailNowf(format string, args ...interface{}) { 245 c.logger.Error(fmt.Sprintf(format, args...)) 246 c.t.FailNow() 247} 248 249// Close shuts down the cluster and cleans up. 250func (c *cluster) Close() { 251 var futures []Future 252 for _, r := range c.rafts { 253 futures = append(futures, r.Shutdown()) 254 } 255 256 // Wait for shutdown 257 limit := time.AfterFunc(c.longstopTimeout, func() { 258 // We can't FailNowf here, and c.Failf won't do anything if we 259 // hang, so panic. 260 panic("timed out waiting for shutdown") 261 }) 262 defer limit.Stop() 263 264 for _, f := range futures { 265 if err := f.Error(); err != nil { 266 c.FailNowf("shutdown future err: %v", err) 267 } 268 } 269 270 for _, d := range c.dirs { 271 os.RemoveAll(d) 272 } 273} 274 275// WaitEventChan returns a channel which will signal if an observation is made 276// or a timeout occurs. It is possible to set a filter to look for specific 277// observations. Setting timeout to 0 means that it will wait forever until a 278// non-filtered observation is made. 279func (c *cluster) WaitEventChan(filter FilterFn, timeout time.Duration) <-chan struct{} { 280 ch := make(chan struct{}) 281 go func() { 282 defer close(ch) 283 var timeoutCh <-chan time.Time 284 if timeout > 0 { 285 timeoutCh = time.After(timeout) 286 } 287 for { 288 select { 289 case <-timeoutCh: 290 return 291 292 case o, ok := <-c.observationCh: 293 if !ok || filter == nil || filter(&o) { 294 return 295 } 296 } 297 } 298 }() 299 return ch 300} 301 302// WaitEvent waits until an observation is made, a timeout occurs, or a test 303// failure is signaled. It is possible to set a filter to look for specific 304// observations. Setting timeout to 0 means that it will wait forever until a 305// non-filtered observation is made or a test failure is signaled. 306func (c *cluster) WaitEvent(filter FilterFn, timeout time.Duration) { 307 select { 308 case <-c.failedCh: 309 c.t.FailNow() 310 311 case <-c.WaitEventChan(filter, timeout): 312 } 313} 314 315// WaitForReplication blocks until every FSM in the cluster has the given 316// length, or the long sanity check timeout expires. 317func (c *cluster) WaitForReplication(fsmLength int) { 318 limitCh := time.After(c.longstopTimeout) 319 320CHECK: 321 for { 322 ch := c.WaitEventChan(nil, c.conf.CommitTimeout) 323 select { 324 case <-c.failedCh: 325 c.t.FailNow() 326 327 case <-limitCh: 328 c.FailNowf("timeout waiting for replication") 329 330 case <-ch: 331 for _, fsmRaw := range c.fsms { 332 fsm := getMockFSM(fsmRaw) 333 fsm.Lock() 334 num := len(fsm.logs) 335 fsm.Unlock() 336 if num != fsmLength { 337 continue CHECK 338 } 339 } 340 return 341 } 342 } 343} 344 345// pollState takes a snapshot of the state of the cluster. This might not be 346// stable, so use GetInState() to apply some additional checks when waiting 347// for the cluster to achieve a particular state. 348func (c *cluster) pollState(s RaftState) ([]*Raft, uint64) { 349 var highestTerm uint64 350 in := make([]*Raft, 0, 1) 351 for _, r := range c.rafts { 352 if r.State() == s { 353 in = append(in, r) 354 } 355 term := r.getCurrentTerm() 356 if term > highestTerm { 357 highestTerm = term 358 } 359 } 360 return in, highestTerm 361} 362 363// GetInState polls the state of the cluster and attempts to identify when it has 364// settled into the given state. 365func (c *cluster) GetInState(s RaftState) []*Raft { 366 c.logger.Info("starting stability test", "raft-state", s) 367 limitCh := time.After(c.longstopTimeout) 368 369 // An election should complete after 2 * max(HeartbeatTimeout, ElectionTimeout) 370 // because of the randomised timer expiring in 1 x interval ... 2 x interval. 371 // We add a bit for propagation delay. If the election fails (e.g. because 372 // two elections start at once), we will have got something through our 373 // observer channel indicating a different state (i.e. one of the nodes 374 // will have moved to candidate state) which will reset the timer. 375 // 376 // Because of an implementation peculiarity, it can actually be 3 x timeout. 377 timeout := c.conf.HeartbeatTimeout 378 if timeout < c.conf.ElectionTimeout { 379 timeout = c.conf.ElectionTimeout 380 } 381 timeout = 2*timeout + c.conf.CommitTimeout 382 timer := time.NewTimer(timeout) 383 defer timer.Stop() 384 385 // Wait until we have a stable instate slice. Each time we see an 386 // observation a state has changed, recheck it and if it has changed, 387 // restart the timer. 388 var pollStartTime = time.Now() 389 for { 390 inState, highestTerm := c.pollState(s) 391 inStateTime := time.Now() 392 393 // Sometimes this routine is called very early on before the 394 // rafts have started up. We then timeout even though no one has 395 // even started an election. So if the highest term in use is 396 // zero, we know there are no raft processes that have yet issued 397 // a RequestVote, and we set a long time out. This is fixed when 398 // we hear the first RequestVote, at which point we reset the 399 // timer. 400 if highestTerm == 0 { 401 timer.Reset(c.longstopTimeout) 402 } else { 403 timer.Reset(timeout) 404 } 405 406 // Filter will wake up whenever we observe a RequestVote. 407 filter := func(ob *Observation) bool { 408 switch ob.Data.(type) { 409 case RaftState: 410 return true 411 case RequestVoteRequest: 412 return true 413 default: 414 return false 415 } 416 } 417 418 select { 419 case <-c.failedCh: 420 c.t.FailNow() 421 422 case <-limitCh: 423 c.FailNowf("timeout waiting for stable %s state", s) 424 425 case <-c.WaitEventChan(filter, 0): 426 c.logger.Debug("resetting stability timeout") 427 428 case t, ok := <-timer.C: 429 if !ok { 430 c.FailNowf("timer channel errored") 431 } 432 433 c.logger.Info(fmt.Sprintf("stable state for %s reached at %s (%d nodes), %s from start of poll, %s from cluster start. Timeout at %s, %s after stability", 434 s, inStateTime, len(inState), inStateTime.Sub(pollStartTime), inStateTime.Sub(c.startTime), t, t.Sub(inStateTime))) 435 return inState 436 } 437 } 438} 439 440// Leader waits for the cluster to elect a leader and stay in a stable state. 441func (c *cluster) Leader() *Raft { 442 leaders := c.GetInState(Leader) 443 if len(leaders) != 1 { 444 c.FailNowf("expected one leader: %v", leaders) 445 } 446 return leaders[0] 447} 448 449// Followers waits for the cluster to have N-1 followers and stay in a stable 450// state. 451func (c *cluster) Followers() []*Raft { 452 expFollowers := len(c.rafts) - 1 453 followers := c.GetInState(Follower) 454 if len(followers) != expFollowers { 455 c.FailNowf("timeout waiting for %d followers (followers are %v)", expFollowers, followers) 456 } 457 return followers 458} 459 460// FullyConnect connects all the transports together. 461func (c *cluster) FullyConnect() { 462 c.logger.Debug("fully connecting") 463 for i, t1 := range c.trans { 464 for j, t2 := range c.trans { 465 if i != j { 466 t1.Connect(t2.LocalAddr(), t2) 467 t2.Connect(t1.LocalAddr(), t1) 468 } 469 } 470 } 471} 472 473// Disconnect disconnects all transports from the given address. 474func (c *cluster) Disconnect(a ServerAddress) { 475 c.logger.Debug("disconnecting", "address", a) 476 for _, t := range c.trans { 477 if t.LocalAddr() == a { 478 t.DisconnectAll() 479 } else { 480 t.Disconnect(a) 481 } 482 } 483} 484 485// Partition keeps the given list of addresses connected but isolates them 486// from the other members of the cluster. 487func (c *cluster) Partition(far []ServerAddress) { 488 c.logger.Debug("partitioning", "addresses", far) 489 490 // Gather the set of nodes on the "near" side of the partition (we 491 // will call the supplied list of nodes the "far" side). 492 near := make(map[ServerAddress]struct{}) 493OUTER: 494 for _, t := range c.trans { 495 l := t.LocalAddr() 496 for _, a := range far { 497 if l == a { 498 continue OUTER 499 } 500 } 501 near[l] = struct{}{} 502 } 503 504 // Now fixup all the connections. The near side will be separated from 505 // the far side, and vice-versa. 506 for _, t := range c.trans { 507 l := t.LocalAddr() 508 if _, ok := near[l]; ok { 509 for _, a := range far { 510 t.Disconnect(a) 511 } 512 } else { 513 for a := range near { 514 t.Disconnect(a) 515 } 516 } 517 } 518} 519 520// IndexOf returns the index of the given raft instance. 521func (c *cluster) IndexOf(r *Raft) int { 522 for i, n := range c.rafts { 523 if n == r { 524 return i 525 } 526 } 527 return -1 528} 529 530// EnsureLeader checks that ALL the nodes think the leader is the given expected 531// leader. 532func (c *cluster) EnsureLeader(t *testing.T, expect ServerAddress) { 533 // We assume c.Leader() has been called already; now check all the rafts 534 // think the leader is correct 535 fail := false 536 for _, r := range c.rafts { 537 leader := ServerAddress(r.Leader()) 538 if leader != expect { 539 if leader == "" { 540 leader = "[none]" 541 } 542 if expect == "" { 543 c.logger.Error("peer sees incorrect leader", "peer", r, "leader", leader, "expected-leader", "[none]") 544 } else { 545 c.logger.Error("peer sees incorrect leader", "peer", r, "leader", leader, "expected-leader", expect) 546 } 547 fail = true 548 } 549 } 550 if fail { 551 c.FailNowf("at least one peer has the wrong notion of leader") 552 } 553} 554 555// EnsureSame makes sure all the FSMs have the same contents. 556func (c *cluster) EnsureSame(t *testing.T) { 557 limit := time.Now().Add(c.longstopTimeout) 558 first := getMockFSM(c.fsms[0]) 559 560CHECK: 561 first.Lock() 562 for i, fsmRaw := range c.fsms { 563 fsm := getMockFSM(fsmRaw) 564 if i == 0 { 565 continue 566 } 567 fsm.Lock() 568 569 if len(first.logs) != len(fsm.logs) { 570 fsm.Unlock() 571 if time.Now().After(limit) { 572 c.FailNowf("FSM log length mismatch: %d %d", 573 len(first.logs), len(fsm.logs)) 574 } else { 575 goto WAIT 576 } 577 } 578 579 for idx := 0; idx < len(first.logs); idx++ { 580 if bytes.Compare(first.logs[idx], fsm.logs[idx]) != 0 { 581 fsm.Unlock() 582 if time.Now().After(limit) { 583 c.FailNowf("FSM log mismatch at index %d", idx) 584 } else { 585 goto WAIT 586 } 587 } 588 } 589 if len(first.configurations) != len(fsm.configurations) { 590 fsm.Unlock() 591 if time.Now().After(limit) { 592 c.FailNowf("FSM configuration length mismatch: %d %d", 593 len(first.logs), len(fsm.logs)) 594 } else { 595 goto WAIT 596 } 597 } 598 599 for idx := 0; idx < len(first.configurations); idx++ { 600 if !reflect.DeepEqual(first.configurations[idx], fsm.configurations[idx]) { 601 fsm.Unlock() 602 if time.Now().After(limit) { 603 c.FailNowf("FSM configuration mismatch at index %d: %v, %v", idx, first.configurations[idx], fsm.configurations[idx]) 604 } else { 605 goto WAIT 606 } 607 } 608 } 609 fsm.Unlock() 610 } 611 612 first.Unlock() 613 return 614 615WAIT: 616 first.Unlock() 617 c.WaitEvent(nil, c.conf.CommitTimeout) 618 goto CHECK 619} 620 621// getConfiguration returns the configuration of the given Raft instance, or 622// fails the test if there's an error 623func (c *cluster) getConfiguration(r *Raft) Configuration { 624 future := r.GetConfiguration() 625 if err := future.Error(); err != nil { 626 c.FailNowf("failed to get configuration: %v", err) 627 return Configuration{} 628 } 629 630 return future.Configuration() 631} 632 633// EnsureSamePeers makes sure all the rafts have the same set of peers. 634func (c *cluster) EnsureSamePeers(t *testing.T) { 635 limit := time.Now().Add(c.longstopTimeout) 636 peerSet := c.getConfiguration(c.rafts[0]) 637 638CHECK: 639 for i, raft := range c.rafts { 640 if i == 0 { 641 continue 642 } 643 644 otherSet := c.getConfiguration(raft) 645 if !reflect.DeepEqual(peerSet, otherSet) { 646 if time.Now().After(limit) { 647 c.FailNowf("peer mismatch: %+v %+v", peerSet, otherSet) 648 } else { 649 goto WAIT 650 } 651 } 652 } 653 return 654 655WAIT: 656 c.WaitEvent(nil, c.conf.CommitTimeout) 657 goto CHECK 658} 659 660// NOTE: This is exposed for middleware testing purposes and is not a stable API 661type MakeClusterOpts struct { 662 Peers int 663 Bootstrap bool 664 Conf *Config 665 ConfigStoreFSM bool 666 MakeFSMFunc func() FSM 667 LongstopTimeout time.Duration 668} 669 670// makeCluster will return a cluster with the given config and number of peers. 671// If bootstrap is true, the servers will know about each other before starting, 672// otherwise their transports will be wired up but they won't yet have configured 673// each other. 674func makeCluster(t *testing.T, opts *MakeClusterOpts) *cluster { 675 if opts.Conf == nil { 676 opts.Conf = inmemConfig(t) 677 } 678 679 c := &cluster{ 680 observationCh: make(chan Observation, 1024), 681 conf: opts.Conf, 682 // Propagation takes a maximum of 2 heartbeat timeouts (time to 683 // get a new heartbeat that would cause a commit) plus a bit. 684 propagateTimeout: opts.Conf.HeartbeatTimeout*2 + opts.Conf.CommitTimeout, 685 longstopTimeout: 5 * time.Second, 686 logger: newTestLoggerWithPrefix(t, "cluster"), 687 failedCh: make(chan struct{}), 688 } 689 if opts.LongstopTimeout > 0 { 690 c.longstopTimeout = opts.LongstopTimeout 691 } 692 693 c.t = t 694 var configuration Configuration 695 696 // Setup the stores and transports 697 for i := 0; i < opts.Peers; i++ { 698 dir, err := ioutil.TempDir("", "raft") 699 if err != nil { 700 c.FailNowf("err: %v", err) 701 } 702 703 store := NewInmemStore() 704 c.dirs = append(c.dirs, dir) 705 c.stores = append(c.stores, store) 706 if opts.ConfigStoreFSM { 707 c.fsms = append(c.fsms, &MockFSMConfigStore{ 708 FSM: &MockFSM{}, 709 }) 710 } else { 711 var fsm FSM 712 if opts.MakeFSMFunc != nil { 713 fsm = opts.MakeFSMFunc() 714 } else { 715 fsm = &MockFSM{} 716 } 717 c.fsms = append(c.fsms, fsm) 718 } 719 720 dir2, snap := FileSnapTest(t) 721 c.dirs = append(c.dirs, dir2) 722 c.snaps = append(c.snaps, snap) 723 724 addr, trans := NewInmemTransport("") 725 c.trans = append(c.trans, trans) 726 localID := ServerID(fmt.Sprintf("server-%s", addr)) 727 if opts.Conf.ProtocolVersion < 3 { 728 localID = ServerID(addr) 729 } 730 configuration.Servers = append(configuration.Servers, Server{ 731 Suffrage: Voter, 732 ID: localID, 733 Address: addr, 734 }) 735 } 736 737 // Wire the transports together 738 c.FullyConnect() 739 740 // Create all the rafts 741 c.startTime = time.Now() 742 for i := 0; i < opts.Peers; i++ { 743 logs := c.stores[i] 744 store := c.stores[i] 745 snap := c.snaps[i] 746 trans := c.trans[i] 747 748 peerConf := opts.Conf 749 peerConf.LocalID = configuration.Servers[i].ID 750 peerConf.Logger = newTestLeveledLoggerWithPrefix(t, string(configuration.Servers[i].ID)) 751 752 if opts.Bootstrap { 753 err := BootstrapCluster(peerConf, logs, store, snap, trans, configuration) 754 if err != nil { 755 c.FailNowf("BootstrapCluster failed: %v", err) 756 } 757 } 758 759 raft, err := NewRaft(peerConf, c.fsms[i], logs, store, snap, trans) 760 if err != nil { 761 c.FailNowf("NewRaft failed: %v", err) 762 } 763 764 raft.RegisterObserver(NewObserver(c.observationCh, false, nil)) 765 if err != nil { 766 c.FailNowf("RegisterObserver failed: %v", err) 767 } 768 c.rafts = append(c.rafts, raft) 769 } 770 771 return c 772} 773 774// NOTE: This is exposed for middleware testing purposes and is not a stable API 775func MakeCluster(n int, t *testing.T, conf *Config) *cluster { 776 return makeCluster(t, &MakeClusterOpts{ 777 Peers: n, 778 Bootstrap: true, 779 Conf: conf, 780 }) 781} 782 783// NOTE: This is exposed for middleware testing purposes and is not a stable API 784func MakeClusterNoBootstrap(n int, t *testing.T, conf *Config) *cluster { 785 return makeCluster(t, &MakeClusterOpts{ 786 Peers: n, 787 Conf: conf, 788 }) 789} 790 791// NOTE: This is exposed for middleware testing purposes and is not a stable API 792func MakeClusterCustom(t *testing.T, opts *MakeClusterOpts) *cluster { 793 return makeCluster(t, opts) 794} 795 796// NOTE: This is exposed for middleware testing purposes and is not a stable API 797func FileSnapTest(t *testing.T) (string, *FileSnapshotStore) { 798 // Create a test dir 799 dir, err := ioutil.TempDir("", "raft") 800 if err != nil { 801 t.Fatalf("err: %v ", err) 802 } 803 804 snap, err := NewFileSnapshotStoreWithLogger(dir, 3, newTestLogger(t)) 805 if err != nil { 806 t.Fatalf("err: %v", err) 807 } 808 return dir, snap 809} 810