1# Check the manual failover 2 3source "../tests/includes/init-tests.tcl" 4 5test "Create a 5 nodes cluster" { 6 create_cluster 5 5 7} 8 9test "Cluster is up" { 10 assert_cluster_state ok 11} 12 13test "Cluster is writable" { 14 cluster_write_test 0 15} 16 17test "Instance #5 is a slave" { 18 assert {[RI 5 role] eq {slave}} 19} 20 21test "Instance #5 synced with the master" { 22 wait_for_condition 1000 50 { 23 [RI 5 master_link_status] eq {up} 24 } else { 25 fail "Instance #5 master link status is not up" 26 } 27} 28 29set current_epoch [CI 1 cluster_current_epoch] 30 31set numkeys 50000 32set numops 10000 33set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]] 34catch {unset content} 35array set content {} 36 37test "Send CLUSTER FAILOVER to #5, during load" { 38 for {set j 0} {$j < $numops} {incr j} { 39 # Write random data to random list. 40 set listid [randomInt $numkeys] 41 set key "key:$listid" 42 set ele [randomValue] 43 # We write both with Lua scripts and with plain commands. 44 # This way we are able to stress Lua -> Redis command invocation 45 # as well, that has tests to prevent Lua to write into wrong 46 # hash slots. 47 if {$listid % 2} { 48 $cluster rpush $key $ele 49 } else { 50 $cluster eval {redis.call("rpush",KEYS[1],ARGV[1])} 1 $key $ele 51 } 52 lappend content($key) $ele 53 54 if {($j % 1000) == 0} { 55 puts -nonewline W; flush stdout 56 } 57 58 if {$j == $numops/2} {R 5 cluster failover} 59 } 60} 61 62test "Wait for failover" { 63 wait_for_condition 1000 50 { 64 [CI 1 cluster_current_epoch] > $current_epoch 65 } else { 66 fail "No failover detected" 67 } 68} 69 70test "Cluster should eventually be up again" { 71 assert_cluster_state ok 72} 73 74test "Cluster is writable" { 75 cluster_write_test 1 76} 77 78test "Instance #5 is now a master" { 79 assert {[RI 5 role] eq {master}} 80} 81 82test "Verify $numkeys keys for consistency with logical content" { 83 # Check that the Redis Cluster content matches our logical content. 84 foreach {key value} [array get content] { 85 assert {[$cluster lrange $key 0 -1] eq $value} 86 } 87} 88 89test "Instance #0 gets converted into a slave" { 90 wait_for_condition 1000 50 { 91 [RI 0 role] eq {slave} 92 } else { 93 fail "Old master was not converted into slave" 94 } 95} 96 97## Check that manual failover does not happen if we can't talk with the master. 98 99source "../tests/includes/init-tests.tcl" 100 101test "Create a 5 nodes cluster" { 102 create_cluster 5 5 103} 104 105test "Cluster is up" { 106 assert_cluster_state ok 107} 108 109test "Cluster is writable" { 110 cluster_write_test 0 111} 112 113test "Instance #5 is a slave" { 114 assert {[RI 5 role] eq {slave}} 115} 116 117test "Instance #5 synced with the master" { 118 wait_for_condition 1000 50 { 119 [RI 5 master_link_status] eq {up} 120 } else { 121 fail "Instance #5 master link status is not up" 122 } 123} 124 125test "Make instance #0 unreachable without killing it" { 126 R 0 deferred 1 127 R 0 DEBUG SLEEP 10 128} 129 130test "Send CLUSTER FAILOVER to instance #5" { 131 R 5 cluster failover 132} 133 134test "Instance #5 is still a slave after some time (no failover)" { 135 after 5000 136 assert {[RI 5 role] eq {master}} 137} 138 139test "Wait for instance #0 to return back alive" { 140 R 0 deferred 0 141 assert {[R 0 read] eq {OK}} 142} 143 144## Check with "force" failover happens anyway. 145 146source "../tests/includes/init-tests.tcl" 147 148test "Create a 5 nodes cluster" { 149 create_cluster 5 5 150} 151 152test "Cluster is up" { 153 assert_cluster_state ok 154} 155 156test "Cluster is writable" { 157 cluster_write_test 0 158} 159 160test "Instance #5 is a slave" { 161 assert {[RI 5 role] eq {slave}} 162} 163 164test "Instance #5 synced with the master" { 165 wait_for_condition 1000 50 { 166 [RI 5 master_link_status] eq {up} 167 } else { 168 fail "Instance #5 master link status is not up" 169 } 170} 171 172test "Make instance #0 unreachable without killing it" { 173 R 0 deferred 1 174 R 0 DEBUG SLEEP 10 175} 176 177test "Send CLUSTER FAILOVER to instance #5" { 178 R 5 cluster failover force 179} 180 181test "Instance #5 is a master after some time" { 182 wait_for_condition 1000 50 { 183 [RI 5 role] eq {master} 184 } else { 185 fail "Instance #5 is not a master after some time regardless of FORCE" 186 } 187} 188 189test "Wait for instance #0 to return back alive" { 190 R 0 deferred 0 191 assert {[R 0 read] eq {OK}} 192} 193