1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.rep.monitor;
9 
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertTrue;
12 
13 import java.util.Arrays;
14 import java.util.concurrent.CountDownLatch;
15 
16 import org.junit.Test;
17 
18 import com.sleepycat.je.rep.NodeType;
19 import com.sleepycat.je.rep.ReplicatedEnvironment;
20 import com.sleepycat.je.rep.impl.node.RepNode;
21 import com.sleepycat.je.rep.monitor.LeaveGroupEvent.LeaveReason;
22 import com.sleepycat.je.rep.utilint.RepTestUtils;
23 import com.sleepycat.je.rep.utilint.RepTestUtils.RepEnvInfo;
24 
25 /**
26  * Test the MonitorChangeListener behaviors.
27  */
28 public class MonitorChangeListenerTest extends MonitorTestBase {
29     private TestChangeListener testListener;
30 
31     /**
32      * Test basic behaviors of MonitorChangeListener.
33      */
34     @Test
testBasicBehaviors()35     public void testBasicBehaviors()
36         throws Exception {
37 
38         checkGroupStart();
39 
40         /*
41          * Close the master, expect to see a NewMasterEvent and a
42          * LeaveGroupEvent.
43          */
44         testListener.masterBarrier = new CountDownLatch(1);
45         testListener.leaveGroupBarrier = new CountDownLatch(1);
46         repEnvInfo[0].closeEnv();
47         /* Wait for a LeaveGroupEvent. */
48         testListener.awaitEvent(testListener.leaveGroupBarrier);
49         /* Wait for elections to settle down. */
50         testListener.awaitEvent(testListener.masterBarrier);
51 
52         /* Do the check. */
53         assertEquals(1, testListener.getMasterEvents());
54         assertEquals(1, testListener.getLeaveGroupEvents());
55         assertTrue(!repEnvInfo[0].getRepConfig().getNodeName().equals
56                    (testListener.masterNodeName));
57 
58         /*
59          * Shutdown all the replicas, see if it generates the expected number
60          * of LeaveGroupEvents.
61          */
62         testListener.clearLeaveGroupEvents();
63         shutdownReplicasNormally();
64     }
65 
66     /* Check the monitor events during the group start up. */
checkGroupStart()67     private void checkGroupStart()
68         throws Exception {
69 
70         repEnvInfo[0].openEnv();
71         RepNode master = repEnvInfo[0].getRepNode();
72         assertTrue(master.isMaster());
73 
74         testListener = new TestChangeListener();
75         testListener.masterBarrier = new CountDownLatch(1);
76         testListener.groupBarrier = new CountDownLatch(1);
77 
78         /*
79          * Start the listener first, so the Listener is guaranteed to get
80          * the group change event when the monitor is registered.
81          */
82 
83         /* generates sync master change event */
84         monitor.startListener(testListener);
85         /* generates async group change event */
86         monitor.register();
87         RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
88 
89         /* Make sure it fires a NewMasterEvent, and do the check. */
90         testListener.awaitEvent(testListener.masterBarrier);
91         assertEquals(1, testListener.getMasterEvents());
92         NewMasterEvent masterEvent = testListener.masterEvent;
93         assertEquals(masterEvent.getNodeName(), master.getNodeName());
94         assertEquals(masterEvent.getMasterName(), master.getNodeName());
95 
96         /* Adding a monitor fires an ADD GroupChangeEvent, do check. */
97         testListener.awaitEvent(testListener.groupBarrier);
98         assertEquals(1, testListener.getGroupAddEvents());
99         GroupChangeEvent groupEvent = testListener.groupEvent;
100         assertEquals(monitor.getNodeName(), groupEvent.getNodeName());
101 
102         /* Get the JoinGroupEvents for current active node: master. */
103         assertEquals(1, testListener.getJoinGroupEvents());
104         JoinGroupEvent joinEvent = testListener.joinEvent;
105         assertEquals(master.getNodeName(), joinEvent.getNodeName());
106         assertEquals(master.getNodeName(), joinEvent.getMasterName());
107 
108         testListener.clearMasterEvents();
109         testListener.clearJoinGroupEvents();
110         testListener.clearGroupAddEvents();
111         for (int i = 1; i < repEnvInfo.length; i++) {
112             testListener.groupBarrier = new CountDownLatch(1);
113             testListener.joinGroupBarrier = new CountDownLatch(1);
114             repEnvInfo[i].openEnv();
115             String nodeName = repEnvInfo[i].getEnv().getNodeName();
116             testListener.awaitEvent(nodeName, testListener.groupBarrier);
117             /* Wait for a JoinGroupEvent. */
118             testListener.awaitEvent(nodeName, testListener.joinGroupBarrier);
119             /* No change in master. */
120             assertEquals(0, testListener.getMasterEvents());
121             assertEquals(i, testListener.getGroupAddEvents());
122             assertEquals(i, testListener.getJoinGroupEvents());
123 
124             /* Do the GroupChangeEvent check. */
125             groupEvent = testListener.groupEvent;
126             assertEquals(nodeName, groupEvent.getNodeName());
127             assertEquals(groupEvent.getRepGroup().getNodes().size(), i + 2);
128 
129             /* Do the JoinGroupEvent check. */
130             joinEvent = testListener.joinEvent;
131             assertEquals(nodeName, joinEvent.getNodeName());
132             assertEquals(master.getNodeName(), joinEvent.getMasterName());
133         }
134     }
135 
136     /*
137      * Shutdown all the replicas normally, do not shutdown the master before
138      * shutting down all replicas so that there is no NewMasterEvent
139      * generated during this process.
140      */
shutdownReplicasNormally()141     private void shutdownReplicasNormally()
142         throws Exception {
143 
144         RepEnvInfo master = null;
145         int shutdownReplicas = 0;
146         for (RepEnvInfo repInfo : repEnvInfo) {
147             ReplicatedEnvironment env = repInfo.getEnv();
148             if ((env == null) || !env.isValid()) {
149                 continue;
150             }
151             if (env.getState().isMaster()) {
152                 master = repInfo;
153                 continue;
154             }
155             shutdownReplicas++;
156             shutdownReplicaAndDoCheck(repInfo, shutdownReplicas);
157         }
158 
159         /* Shutdown the master. */
160         if (master != null) {
161             shutdownReplicas++;
162             shutdownReplicaAndDoCheck(master, shutdownReplicas);
163         }
164     }
165 
166     /* Shutdown a replica and do the check. */
shutdownReplicaAndDoCheck(RepEnvInfo replica, int index)167     private void shutdownReplicaAndDoCheck(RepEnvInfo replica,
168                                            int index)
169         throws Exception {
170 
171         testListener.leaveGroupBarrier = new CountDownLatch(1);
172         String nodeName = replica.getEnv().getNodeName();
173         replica.closeEnv();
174         testListener.awaitEvent(nodeName, testListener.leaveGroupBarrier);
175 
176         /* Do the check. */
177         LeaveGroupEvent event = testListener.leaveEvent;
178         assertEquals(index, testListener.getLeaveGroupEvents());
179         assertEquals(nodeName, event.getNodeName());
180 
181         checkShutdownReplicaLeaveReason(event);
182     }
183 
checkShutdownReplicaLeaveReason(final LeaveGroupEvent event)184     void checkShutdownReplicaLeaveReason(final LeaveGroupEvent event) {
185         assertEquals(LeaveReason.NORMAL_SHUTDOWN, event.getLeaveReason());
186     }
187 
188     /**
189      * Test removeMember which would create GroupChangeEvent, but no
190      * LeaveGroupEvents.
191      */
192     @Test
testRemoveMember()193     public void testRemoveMember()
194         throws Exception {
195 
196         checkGroupStart();
197 
198         RepNode master = repEnvInfo[0].getRepNode();
199         assertTrue(master.isMaster());
200 
201         /*
202          * Remove replica from RepGroupDB, see if it fires a REMOVE
203          * GroupChangeEvent.
204          */
205         testListener.clearGroupAddEvents();
206         testListener.clearLeaveGroupEvents();
207         for (int i = 1; i < repEnvInfo.length; i++) {
208             testListener.groupBarrier = new CountDownLatch(1);
209             String nodeName = repEnvInfo[i].getRepNode().getNodeName();
210             master.removeMember(nodeName);
211             testListener.awaitEvent(nodeName, testListener.groupBarrier);
212             assertEquals(0, testListener.getGroupAddEvents());
213             assertEquals(i, testListener.getGroupRemoveEvents());
214             assertEquals(nodeName, testListener.groupEvent.getNodeName());
215         }
216         assertEquals(0, testListener.getLeaveGroupEvents());
217 
218         /*
219          * Shutdown all the replicas, see if it generates the expected number
220          * of LeaveGroupEvents.
221          */
222         shutdownReplicasNormally();
223     }
224 
225     @Test
testActiveNodesWhenMonitorStarts()226     public void testActiveNodesWhenMonitorStarts()
227         throws Exception {
228 
229         RepTestUtils.joinGroup(repEnvInfo);
230         testListener = new TestChangeListener();
231         /* generates sync master change event */
232         monitor.startListener(testListener);
233         /* generates async group change event */
234         monitor.register();
235         RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
236 
237         assertEquals(1, testListener.getMasterEvents());
238         assertEquals(5, testListener.getJoinGroupEvents());
239         JoinGroupEvent event = testListener.joinEvent;
240         assertEquals
241             (repEnvInfo[0].getEnv().getNodeName(), event.getMasterName());
242 
243         shutdownReplicasNormally();
244     }
245 
246     /**
247      * Test monitor events when adding a secondary node.
248      */
249     @Test
testAddSecondaryNode()250     public void testAddSecondaryNode()
251         throws Exception {
252 
253         checkGroupStart();
254         final RepNode master = repEnvInfo[0].getRepNode();
255         assertTrue("Master", master.isMaster());
256 
257         testListener.clearGroupAddEvents();
258         testListener.clearGroupRemoveEvents();
259         testListener.clearJoinGroupEvents();
260         testListener.clearLeaveGroupEvents();
261         testListener.joinGroupBarrier = new CountDownLatch(1);
262 
263         /* Create a new secondary */
264         final int pos = repEnvInfo.length;
265         repEnvInfo = Arrays.copyOf(repEnvInfo, pos + 1);
266         repEnvInfo[pos] = RepTestUtils.setupEnvInfo(
267             RepTestUtils.makeRepEnvDir(envRoot, pos),
268             RepTestUtils.createEnvConfig(RepTestUtils.DEFAULT_DURABILITY),
269             RepTestUtils.createRepConfig(pos + 1).setNodeType(
270                 NodeType.SECONDARY),
271             repEnvInfo[0]);
272         repEnvInfo[pos].openEnv();
273         final String nodeName = repEnvInfo[pos].getEnv().getNodeName();
274 
275         testListener.awaitEvent(testListener.joinGroupBarrier);
276         assertEquals("Add events", 0, testListener.getGroupAddEvents());
277         assertEquals("Remove events", 0, testListener.getGroupRemoveEvents());
278         assertEquals("Join events", 1, testListener.getJoinGroupEvents());
279         assertEquals("Join event node name",
280                      nodeName, testListener.joinEvent.getNodeName());
281         assertEquals("Leave events", 0, testListener.getLeaveGroupEvents());
282 
283         testListener.clearGroupAddEvents();
284         testListener.clearGroupRemoveEvents();
285         testListener.clearJoinGroupEvents();
286         testListener.clearLeaveGroupEvents();
287         testListener.leaveGroupBarrier = new CountDownLatch(1);
288 
289         /* Close secondary */
290         repEnvInfo[pos].closeEnv();
291 
292         testListener.awaitEvent(testListener.leaveGroupBarrier);
293         assertEquals("Add events", 0, testListener.getGroupAddEvents());
294         assertEquals("Remove events", 0, testListener.getGroupRemoveEvents());
295         assertEquals("Join events", 0, testListener.getJoinGroupEvents());
296         assertEquals("Leave events", 1, testListener.getLeaveGroupEvents());
297         assertEquals("Leave event node name",
298                      nodeName, testListener.leaveEvent.getNodeName());
299         testListener.clearLeaveGroupEvents();
300 
301         /* Shutdown */
302         shutdownReplicasNormally();
303     }
304 
305     /**
306      * Test monitor events when replacing a primary replica with a secondary
307      * node that reuses its environment.
308      */
309     @Test
testReplaceAsSecondaryNode()310     public void testReplaceAsSecondaryNode()
311         throws Exception {
312 
313         checkGroupStart();
314         final RepNode master = repEnvInfo[0].getRepNode();
315         assertTrue("Master", master.isMaster());
316 
317         /* Remove replica */
318         testListener.clearGroupAddEvents();
319         testListener.clearGroupRemoveEvents();
320         testListener.clearJoinGroupEvents();
321         testListener.clearLeaveGroupEvents();
322         testListener.groupBarrier = new CountDownLatch(1);
323         testListener.leaveGroupBarrier = new CountDownLatch(1);
324         master.removeMember(repEnvInfo[1].getRepNode().getNodeName());
325         testListener.awaitEvent(testListener.groupBarrier);
326         assertEquals("Add events", 0, testListener.getGroupAddEvents());
327         assertEquals("Remove events", 1, testListener.getGroupRemoveEvents());
328         assertEquals("Remove event node name",
329                      repEnvInfo[1].getEnv().getNodeName(),
330                      testListener.groupEvent.getNodeName());
331         assertEquals("Join events", 0, testListener.getJoinGroupEvents());
332         assertEquals("Leave events", 0, testListener.getLeaveGroupEvents());
333 
334         /* Close environment */
335         String closedName = repEnvInfo[1].getEnv().getNodeName();
336         repEnvInfo[1].closeEnv();
337         testListener.awaitEvent(testListener.leaveGroupBarrier);
338         assertEquals(closedName, testListener.leaveEvent.getNodeName());
339 
340         /* Open replica as a (new) secondary */
341         testListener.clearGroupAddEvents();
342         testListener.clearGroupRemoveEvents();
343         testListener.clearJoinGroupEvents();
344         testListener.clearLeaveGroupEvents();
345         testListener.joinGroupBarrier = new CountDownLatch(1);
346         repEnvInfo[1].getRepConfig()
347             .setNodeName("Node2-secondary")
348             .setNodeType(NodeType.SECONDARY);
349         repEnvInfo[1].openEnv();
350         testListener.awaitEvent(testListener.joinGroupBarrier);
351         assertEquals("Add events", 0, testListener.getGroupAddEvents());
352         assertEquals("Remove events", 0, testListener.getGroupRemoveEvents());
353         assertEquals("Join events", 1, testListener.getJoinGroupEvents());
354         assertEquals("Join event node name",
355                      repEnvInfo[1].getEnv().getNodeName(),
356                      testListener.joinEvent.getNodeName());
357         assertEquals("Leave events", 0, testListener.getLeaveGroupEvents());
358 
359         /* Shutdown */
360         shutdownReplicasNormally();
361     }
362 }
363