1# Check the manual failover
2
3source "../tests/includes/init-tests.tcl"
4
5test "Create a 5 nodes cluster" {
6    create_cluster 5 5
7}
8
9test "Cluster is up" {
10    assert_cluster_state ok
11}
12
13test "Cluster is writable" {
14    cluster_write_test 0
15}
16
17test "Instance #5 is a slave" {
18    assert {[RI 5 role] eq {slave}}
19}
20
21test "Instance #5 synced with the master" {
22    wait_for_condition 1000 50 {
23        [RI 5 master_link_status] eq {up}
24    } else {
25        fail "Instance #5 master link status is not up"
26    }
27}
28
29set current_epoch [CI 1 cluster_current_epoch]
30
31set numkeys 50000
32set numops 10000
33set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
34catch {unset content}
35array set content {}
36
37test "Send CLUSTER FAILOVER to #5, during load" {
38    for {set j 0} {$j < $numops} {incr j} {
39        # Write random data to random list.
40        set listid [randomInt $numkeys]
41        set key "key:$listid"
42        set ele [randomValue]
43        # We write both with Lua scripts and with plain commands.
44        # This way we are able to stress Lua -> Redis command invocation
45        # as well, that has tests to prevent Lua to write into wrong
46        # hash slots.
47        if {$listid % 2} {
48            $cluster rpush $key $ele
49        } else {
50           $cluster eval {redis.call("rpush",KEYS[1],ARGV[1])} 1 $key $ele
51        }
52        lappend content($key) $ele
53
54        if {($j % 1000) == 0} {
55            puts -nonewline W; flush stdout
56        }
57
58        if {$j == $numops/2} {R 5 cluster failover}
59    }
60}
61
62test "Wait for failover" {
63    wait_for_condition 1000 50 {
64        [CI 1 cluster_current_epoch] > $current_epoch
65    } else {
66        fail "No failover detected"
67    }
68}
69
70test "Cluster should eventually be up again" {
71    assert_cluster_state ok
72}
73
74test "Cluster is writable" {
75    cluster_write_test 1
76}
77
78test "Instance #5 is now a master" {
79    assert {[RI 5 role] eq {master}}
80}
81
82test "Verify $numkeys keys for consistency with logical content" {
83    # Check that the Redis Cluster content matches our logical content.
84    foreach {key value} [array get content] {
85        assert {[$cluster lrange $key 0 -1] eq $value}
86    }
87}
88
89test "Instance #0 gets converted into a slave" {
90    wait_for_condition 1000 50 {
91        [RI 0 role] eq {slave}
92    } else {
93        fail "Old master was not converted into slave"
94    }
95}
96
97## Check that manual failover does not happen if we can't talk with the master.
98
99source "../tests/includes/init-tests.tcl"
100
101test "Create a 5 nodes cluster" {
102    create_cluster 5 5
103}
104
105test "Cluster is up" {
106    assert_cluster_state ok
107}
108
109test "Cluster is writable" {
110    cluster_write_test 0
111}
112
113test "Instance #5 is a slave" {
114    assert {[RI 5 role] eq {slave}}
115}
116
117test "Instance #5 synced with the master" {
118    wait_for_condition 1000 50 {
119        [RI 5 master_link_status] eq {up}
120    } else {
121        fail "Instance #5 master link status is not up"
122    }
123}
124
125test "Make instance #0 unreachable without killing it" {
126    R 0 deferred 1
127    R 0 DEBUG SLEEP 10
128}
129
130test "Send CLUSTER FAILOVER to instance #5" {
131    R 5 cluster failover
132}
133
134test "Instance #5 is still a slave after some time (no failover)" {
135    after 5000
136    assert {[RI 5 role] eq {master}}
137}
138
139test "Wait for instance #0 to return back alive" {
140    R 0 deferred 0
141    assert {[R 0 read] eq {OK}}
142}
143
144## Check with "force" failover happens anyway.
145
146source "../tests/includes/init-tests.tcl"
147
148test "Create a 5 nodes cluster" {
149    create_cluster 5 5
150}
151
152test "Cluster is up" {
153    assert_cluster_state ok
154}
155
156test "Cluster is writable" {
157    cluster_write_test 0
158}
159
160test "Instance #5 is a slave" {
161    assert {[RI 5 role] eq {slave}}
162}
163
164test "Instance #5 synced with the master" {
165    wait_for_condition 1000 50 {
166        [RI 5 master_link_status] eq {up}
167    } else {
168        fail "Instance #5 master link status is not up"
169    }
170}
171
172test "Make instance #0 unreachable without killing it" {
173    R 0 deferred 1
174    R 0 DEBUG SLEEP 10
175}
176
177test "Send CLUSTER FAILOVER to instance #5" {
178    R 5 cluster failover force
179}
180
181test "Instance #5 is a master after some time" {
182    wait_for_condition 1000 50 {
183        [RI 5 role] eq {master}
184    } else {
185        fail "Instance #5 is not a master after some time regardless of FORCE"
186    }
187}
188
189test "Wait for instance #0 to return back alive" {
190    R 0 deferred 0
191    assert {[R 0 read] eq {OK}}
192}
193