1 /**
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *     http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.apache.hadoop.hbase.master;
20 
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertNotEquals;
23 import static org.junit.Assert.assertTrue;
24 
25 import java.util.List;
26 import java.util.Map;
27 
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.MiniHBaseCluster;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableExistsException;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.Connection;
39 import org.apache.hadoop.hbase.client.ConnectionFactory;
40 import org.apache.hadoop.hbase.client.MetaScanner;
41 import org.apache.hadoop.hbase.executor.EventType;
42 import org.apache.hadoop.hbase.testclassification.LargeTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.JVMClusterUtil;
45 import org.apache.hadoop.hbase.util.Threads;
46 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
47 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
48 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
49 import org.junit.After;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52 
53 @Category(LargeTests.class)
54 public class TestRestartCluster {
55   private static final Log LOG = LogFactory.getLog(TestRestartCluster.class);
56   private HBaseTestingUtility UTIL = new HBaseTestingUtility();
57 
58   private static final byte[] TABLENAME = Bytes.toBytes("master_transitions");
59   private static final byte [][] FAMILIES = {Bytes.toBytes("a")};
60   private static final TableName[] TABLES = {
61       TableName.valueOf("restartTableOne"),
62       TableName.valueOf("restartTableTwo"),
63       TableName.valueOf("restartTableThree")
64   };
65   private static final byte [] FAMILY = Bytes.toBytes("family");
66 
tearDown()67   @After public void tearDown() throws Exception {
68     UTIL.shutdownMiniCluster();
69   }
70 
testRestartClusterAfterKill()71   @Test (timeout=300000) public void testRestartClusterAfterKill()
72   throws Exception {
73     UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
74     UTIL.startMiniZKCluster();
75     ZooKeeperWatcher zooKeeper =
76       new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null, true);
77 
78     // create the unassigned region, throw up a region opened state for META
79     String unassignedZNode = zooKeeper.assignmentZNode;
80     ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode);
81 
82     ServerName sn = ServerName.valueOf(HMaster.MASTER, 1, System.currentTimeMillis());
83 
84     ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO, sn);
85 
86     LOG.debug("Created UNASSIGNED zNode for ROOT and hbase:meta regions in state " +
87         EventType.M_ZK_REGION_OFFLINE);
88 
89     // start the HB cluster
90     LOG.info("Starting HBase cluster...");
91     UTIL.startMiniCluster(2);
92 
93     UTIL.createTable(TABLENAME, FAMILIES);
94     LOG.info("Created a table, waiting for table to be available...");
95     UTIL.waitTableAvailable(TABLENAME, 60*1000);
96 
97     LOG.info("Master deleted unassigned region and started up successfully.");
98   }
99 
100   @Test (timeout=300000)
testClusterRestart()101   public void testClusterRestart() throws Exception {
102     UTIL.startMiniCluster(3);
103     Connection connection = UTIL.getConnection();
104 
105     while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
106       Threads.sleep(1);
107     }
108     LOG.info("\n\nCreating tables");
109     for(TableName TABLE : TABLES) {
110       UTIL.createTable(TABLE, FAMILY);
111     }
112     for(TableName TABLE : TABLES) {
113       UTIL.waitTableEnabled(TABLE);
114     }
115 
116     List<HRegionInfo> allRegions =
117         MetaScanner.listAllRegions(UTIL.getConfiguration(), connection, true);
118     assertEquals(4, allRegions.size());
119 
120     LOG.info("\n\nShutting down cluster");
121     UTIL.shutdownMiniHBaseCluster();
122 
123     LOG.info("\n\nSleeping a bit");
124     Thread.sleep(2000);
125 
126     LOG.info("\n\nStarting cluster the second time");
127     UTIL.restartHBaseCluster(3);
128 
129     // Need to use a new 'Configuration' so we make a new HConnection.
130     // Otherwise we're reusing an HConnection that has gone stale because
131     // the shutdown of the cluster also called shut of the connection.
132     allRegions =
133         MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), connection, true);
134     assertEquals(4, allRegions.size());
135     LOG.info("\n\nWaiting for tables to be available");
136     for(TableName TABLE: TABLES) {
137       try {
138         UTIL.createTable(TABLE, FAMILY);
139         assertTrue("Able to create table that should already exist", false);
140       } catch(TableExistsException tee) {
141         LOG.info("Table already exists as expected");
142       }
143       UTIL.waitTableAvailable(TABLE);
144     }
145   }
146 
147   /**
148    * This tests retaining assignments on a cluster restart
149    */
150   @Test (timeout=300000)
testRetainAssignmentOnRestart()151   public void testRetainAssignmentOnRestart() throws Exception {
152     UTIL.startMiniCluster(2);
153     while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
154       Threads.sleep(1);
155     }
156     // Turn off balancer
157     UTIL.getMiniHBaseCluster().getMaster().
158       getMasterRpcServices().synchronousBalanceSwitch(false);
159     LOG.info("\n\nCreating tables");
160     for(TableName TABLE : TABLES) {
161       UTIL.createTable(TABLE, FAMILY);
162     }
163     for(TableName TABLE : TABLES) {
164       UTIL.waitTableEnabled(TABLE);
165     }
166 
167     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
168     AssignmentManager am = master.getAssignmentManager();
169     am.waitUntilNoRegionsInTransition(120000);
170 
171     // We don't have to use SnapshotOfRegionAssignmentFromMeta.
172     // We use it here because AM used to use it to load all user region placements
173     SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
174       master.getConnection());
175     snapshot.initialize();
176     Map<HRegionInfo, ServerName> regionToRegionServerMap
177       = snapshot.getRegionToRegionServerMap();
178 
179     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
180     List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
181     assertEquals(2, threads.size());
182     int[] rsPorts = new int[2];
183     for (int i = 0; i < 2; i++) {
184       rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
185     }
186     for (ServerName serverName: regionToRegionServerMap.values()) {
187       boolean found = false; // Test only, no need to optimize
188       for (int k = 0; k < 2 && !found; k++) {
189         found = serverName.getPort() == rsPorts[k];
190       }
191       assertTrue(found);
192     }
193 
194     LOG.info("\n\nShutting down HBase cluster");
195     cluster.shutdown();
196     cluster.waitUntilShutDown();
197 
198     LOG.info("\n\nSleeping a bit");
199     Thread.sleep(2000);
200 
201     LOG.info("\n\nStarting cluster the second time with the same ports");
202     try {
203       cluster.getConf().setInt(
204         ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
205       master = cluster.startMaster().getMaster();
206       for (int i = 0; i < 2; i++) {
207         cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
208         cluster.startRegionServer();
209       }
210     } finally {
211       // Reset region server port so as not to conflict with other tests
212       cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
213       cluster.getConf().setInt(
214         ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
215     }
216 
217     // Make sure live regionservers are on the same host/port
218     List<ServerName> localServers = master.getServerManager().getOnlineServersList();
219     assertEquals(2, localServers.size());
220     for (int i = 0; i < 2; i++) {
221       boolean found = false;
222       for (ServerName serverName: localServers) {
223         if (serverName.getPort() == rsPorts[i]) {
224           found = true;
225           break;
226         }
227       }
228       assertTrue(found);
229     }
230 
231     // Wait till master is initialized and all regions are assigned
232     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
233     int expectedRegions = regionToRegionServerMap.size() + 1;
234     while (!master.isInitialized()
235         || regionStates.getRegionAssignments().size() != expectedRegions) {
236       Threads.sleep(100);
237     }
238 
239     snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection());
240     snapshot.initialize();
241     Map<HRegionInfo, ServerName> newRegionToRegionServerMap =
242       snapshot.getRegionToRegionServerMap();
243     assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
244     for (Map.Entry<HRegionInfo, ServerName> entry: newRegionToRegionServerMap.entrySet()) {
245       if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
246       ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
247       ServerName currentServer = entry.getValue();
248       assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
249       assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
250     }
251   }
252 }
253