1 /** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 package org.apache.hadoop.hbase.master; 20 21 import static org.junit.Assert.assertEquals; 22 import static org.junit.Assert.assertNotEquals; 23 import static org.junit.Assert.assertTrue; 24 25 import java.util.List; 26 import java.util.Map; 27 28 import org.apache.commons.logging.Log; 29 import org.apache.commons.logging.LogFactory; 30 import org.apache.hadoop.conf.Configuration; 31 import org.apache.hadoop.hbase.HBaseTestingUtility; 32 import org.apache.hadoop.hbase.HConstants; 33 import org.apache.hadoop.hbase.HRegionInfo; 34 import org.apache.hadoop.hbase.MiniHBaseCluster; 35 import org.apache.hadoop.hbase.ServerName; 36 import org.apache.hadoop.hbase.TableExistsException; 37 import org.apache.hadoop.hbase.TableName; 38 import org.apache.hadoop.hbase.client.Connection; 39 import org.apache.hadoop.hbase.client.ConnectionFactory; 40 import org.apache.hadoop.hbase.client.MetaScanner; 41 import org.apache.hadoop.hbase.executor.EventType; 42 import org.apache.hadoop.hbase.testclassification.LargeTests; 43 import org.apache.hadoop.hbase.util.Bytes; 44 import org.apache.hadoop.hbase.util.JVMClusterUtil; 45 import org.apache.hadoop.hbase.util.Threads; 46 import org.apache.hadoop.hbase.zookeeper.ZKAssign; 47 import org.apache.hadoop.hbase.zookeeper.ZKUtil; 48 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; 49 import org.junit.After; 50 import org.junit.Test; 51 import org.junit.experimental.categories.Category; 52 53 @Category(LargeTests.class) 54 public class TestRestartCluster { 55 private static final Log LOG = LogFactory.getLog(TestRestartCluster.class); 56 private HBaseTestingUtility UTIL = new HBaseTestingUtility(); 57 58 private static final byte[] TABLENAME = Bytes.toBytes("master_transitions"); 59 private static final byte [][] FAMILIES = {Bytes.toBytes("a")}; 60 private static final TableName[] TABLES = { 61 TableName.valueOf("restartTableOne"), 62 TableName.valueOf("restartTableTwo"), 63 TableName.valueOf("restartTableThree") 64 }; 65 private static final byte [] FAMILY = Bytes.toBytes("family"); 66 tearDown()67 @After public void tearDown() throws Exception { 68 UTIL.shutdownMiniCluster(); 69 } 70 testRestartClusterAfterKill()71 @Test (timeout=300000) public void testRestartClusterAfterKill() 72 throws Exception { 73 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); 74 UTIL.startMiniZKCluster(); 75 ZooKeeperWatcher zooKeeper = 76 new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null, true); 77 78 // create the unassigned region, throw up a region opened state for META 79 String unassignedZNode = zooKeeper.assignmentZNode; 80 ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode); 81 82 ServerName sn = ServerName.valueOf(HMaster.MASTER, 1, System.currentTimeMillis()); 83 84 ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO, sn); 85 86 LOG.debug("Created UNASSIGNED zNode for ROOT and hbase:meta regions in state " + 87 EventType.M_ZK_REGION_OFFLINE); 88 89 // start the HB cluster 90 LOG.info("Starting HBase cluster..."); 91 UTIL.startMiniCluster(2); 92 93 UTIL.createTable(TABLENAME, FAMILIES); 94 LOG.info("Created a table, waiting for table to be available..."); 95 UTIL.waitTableAvailable(TABLENAME, 60*1000); 96 97 LOG.info("Master deleted unassigned region and started up successfully."); 98 } 99 100 @Test (timeout=300000) testClusterRestart()101 public void testClusterRestart() throws Exception { 102 UTIL.startMiniCluster(3); 103 Connection connection = UTIL.getConnection(); 104 105 while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { 106 Threads.sleep(1); 107 } 108 LOG.info("\n\nCreating tables"); 109 for(TableName TABLE : TABLES) { 110 UTIL.createTable(TABLE, FAMILY); 111 } 112 for(TableName TABLE : TABLES) { 113 UTIL.waitTableEnabled(TABLE); 114 } 115 116 List<HRegionInfo> allRegions = 117 MetaScanner.listAllRegions(UTIL.getConfiguration(), connection, true); 118 assertEquals(4, allRegions.size()); 119 120 LOG.info("\n\nShutting down cluster"); 121 UTIL.shutdownMiniHBaseCluster(); 122 123 LOG.info("\n\nSleeping a bit"); 124 Thread.sleep(2000); 125 126 LOG.info("\n\nStarting cluster the second time"); 127 UTIL.restartHBaseCluster(3); 128 129 // Need to use a new 'Configuration' so we make a new HConnection. 130 // Otherwise we're reusing an HConnection that has gone stale because 131 // the shutdown of the cluster also called shut of the connection. 132 allRegions = 133 MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), connection, true); 134 assertEquals(4, allRegions.size()); 135 LOG.info("\n\nWaiting for tables to be available"); 136 for(TableName TABLE: TABLES) { 137 try { 138 UTIL.createTable(TABLE, FAMILY); 139 assertTrue("Able to create table that should already exist", false); 140 } catch(TableExistsException tee) { 141 LOG.info("Table already exists as expected"); 142 } 143 UTIL.waitTableAvailable(TABLE); 144 } 145 } 146 147 /** 148 * This tests retaining assignments on a cluster restart 149 */ 150 @Test (timeout=300000) testRetainAssignmentOnRestart()151 public void testRetainAssignmentOnRestart() throws Exception { 152 UTIL.startMiniCluster(2); 153 while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { 154 Threads.sleep(1); 155 } 156 // Turn off balancer 157 UTIL.getMiniHBaseCluster().getMaster(). 158 getMasterRpcServices().synchronousBalanceSwitch(false); 159 LOG.info("\n\nCreating tables"); 160 for(TableName TABLE : TABLES) { 161 UTIL.createTable(TABLE, FAMILY); 162 } 163 for(TableName TABLE : TABLES) { 164 UTIL.waitTableEnabled(TABLE); 165 } 166 167 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); 168 AssignmentManager am = master.getAssignmentManager(); 169 am.waitUntilNoRegionsInTransition(120000); 170 171 // We don't have to use SnapshotOfRegionAssignmentFromMeta. 172 // We use it here because AM used to use it to load all user region placements 173 SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta( 174 master.getConnection()); 175 snapshot.initialize(); 176 Map<HRegionInfo, ServerName> regionToRegionServerMap 177 = snapshot.getRegionToRegionServerMap(); 178 179 MiniHBaseCluster cluster = UTIL.getHBaseCluster(); 180 List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads(); 181 assertEquals(2, threads.size()); 182 int[] rsPorts = new int[2]; 183 for (int i = 0; i < 2; i++) { 184 rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort(); 185 } 186 for (ServerName serverName: regionToRegionServerMap.values()) { 187 boolean found = false; // Test only, no need to optimize 188 for (int k = 0; k < 2 && !found; k++) { 189 found = serverName.getPort() == rsPorts[k]; 190 } 191 assertTrue(found); 192 } 193 194 LOG.info("\n\nShutting down HBase cluster"); 195 cluster.shutdown(); 196 cluster.waitUntilShutDown(); 197 198 LOG.info("\n\nSleeping a bit"); 199 Thread.sleep(2000); 200 201 LOG.info("\n\nStarting cluster the second time with the same ports"); 202 try { 203 cluster.getConf().setInt( 204 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2); 205 master = cluster.startMaster().getMaster(); 206 for (int i = 0; i < 2; i++) { 207 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]); 208 cluster.startRegionServer(); 209 } 210 } finally { 211 // Reset region server port so as not to conflict with other tests 212 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0); 213 cluster.getConf().setInt( 214 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); 215 } 216 217 // Make sure live regionservers are on the same host/port 218 List<ServerName> localServers = master.getServerManager().getOnlineServersList(); 219 assertEquals(2, localServers.size()); 220 for (int i = 0; i < 2; i++) { 221 boolean found = false; 222 for (ServerName serverName: localServers) { 223 if (serverName.getPort() == rsPorts[i]) { 224 found = true; 225 break; 226 } 227 } 228 assertTrue(found); 229 } 230 231 // Wait till master is initialized and all regions are assigned 232 RegionStates regionStates = master.getAssignmentManager().getRegionStates(); 233 int expectedRegions = regionToRegionServerMap.size() + 1; 234 while (!master.isInitialized() 235 || regionStates.getRegionAssignments().size() != expectedRegions) { 236 Threads.sleep(100); 237 } 238 239 snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); 240 snapshot.initialize(); 241 Map<HRegionInfo, ServerName> newRegionToRegionServerMap = 242 snapshot.getRegionToRegionServerMap(); 243 assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size()); 244 for (Map.Entry<HRegionInfo, ServerName> entry: newRegionToRegionServerMap.entrySet()) { 245 if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue; 246 ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); 247 ServerName currentServer = entry.getValue(); 248 assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort()); 249 assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); 250 } 251 } 252 } 253