1 /** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 package org.apache.hadoop.hbase.util; 20 21 import static org.junit.Assert.assertTrue; 22 23 import java.io.IOException; 24 import java.util.List; 25 26 import org.apache.commons.logging.Log; 27 import org.apache.commons.logging.LogFactory; 28 import org.apache.hadoop.conf.Configuration; 29 import org.apache.hadoop.fs.FileSystem; 30 import org.apache.hadoop.fs.Path; 31 import org.apache.hadoop.hbase.*; 32 import org.apache.hadoop.hbase.client.Admin; 33 import org.apache.hadoop.hbase.client.Connection; 34 import org.apache.hadoop.hbase.client.HBaseAdmin; 35 import org.apache.hadoop.hbase.client.HConnectionManager; 36 import org.apache.hadoop.hbase.client.Put; 37 import org.apache.hadoop.hbase.client.Durability; 38 import org.apache.hadoop.hbase.regionserver.HRegion; 39 import org.apache.hadoop.hbase.testclassification.MediumTests; 40 import org.junit.Test; 41 import org.junit.experimental.categories.Category; 42 43 /** 44 * Tests merging a normal table's regions 45 */ 46 @Category(MediumTests.class) 47 public class TestMergeTable { 48 private static final Log LOG = LogFactory.getLog(TestMergeTable.class); 49 private final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 50 private static final byte [] COLUMN_NAME = Bytes.toBytes("contents"); 51 private static final byte [] VALUE; 52 static { 53 // We will use the same value for the rows as that is not really important here 54 String partialValue = String.valueOf(System.currentTimeMillis()); 55 StringBuilder val = new StringBuilder(); 56 while (val.length() < 1024) { 57 val.append(partialValue); 58 } 59 VALUE = Bytes.toBytes(val.toString()); 60 } 61 62 /** 63 * Test merge. 64 * Hand-makes regions of a mergeable size and adds the hand-made regions to 65 * hand-made meta. The hand-made regions are created offline. We then start 66 * up mini cluster, disables the hand-made table and starts in on merging. 67 * @throws Exception 68 */ testMergeTable()69 @Test (timeout=300000) public void testMergeTable() throws Exception { 70 // Table we are manually creating offline. 71 HTableDescriptor desc = new HTableDescriptor(org.apache.hadoop.hbase.TableName.valueOf(Bytes.toBytes("test"))); 72 desc.addFamily(new HColumnDescriptor(COLUMN_NAME)); 73 74 // Set maximum regionsize down. 75 UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, 64L * 1024L * 1024L); 76 // Make it so we don't split. 77 UTIL.getConfiguration().setInt("hbase.regionserver.regionSplitLimit", 0); 78 // Startup hdfs. Its in here we'll be putting our manually made regions. 79 UTIL.startMiniDFSCluster(1); 80 // Create hdfs hbase rootdir. 81 Path rootdir = UTIL.createRootDir(); 82 FileSystem fs = FileSystem.get(UTIL.getConfiguration()); 83 if (fs.exists(rootdir)) { 84 if (fs.delete(rootdir, true)) { 85 LOG.info("Cleaned up existing " + rootdir); 86 } 87 } 88 89 // Now create three data regions: The first is too large to merge since it 90 // will be > 64 MB in size. The second two will be smaller and will be 91 // selected for merging. 92 93 // To ensure that the first region is larger than 64MB we need to write at 94 // least 65536 rows. We will make certain by writing 70000 95 byte [] row_70001 = Bytes.toBytes("row_70001"); 96 byte [] row_80001 = Bytes.toBytes("row_80001"); 97 98 // Create regions and populate them at same time. Create the tabledir 99 // for them first. 100 new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir).createTableDescriptor(desc); 101 HRegion [] regions = { 102 createRegion(desc, null, row_70001, 1, 70000, rootdir), 103 createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir), 104 createRegion(desc, row_80001, null, 80001, 11000, rootdir) 105 }; 106 107 // Now create the root and meta regions and insert the data regions 108 // created above into hbase:meta 109 setupMeta(rootdir, regions); 110 try { 111 LOG.info("Starting mini zk cluster"); 112 UTIL.startMiniZKCluster(); 113 LOG.info("Starting mini hbase cluster"); 114 UTIL.startMiniHBaseCluster(1, 1); 115 Configuration c = new Configuration(UTIL.getConfiguration()); 116 Connection connection = HConnectionManager.getConnection(c); 117 118 List<HRegionInfo> originalTableRegions = 119 MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(), connection, 120 desc.getTableName()); 121 LOG.info("originalTableRegions size=" + originalTableRegions.size() + 122 "; " + originalTableRegions); 123 Admin admin = new HBaseAdmin(c); 124 admin.disableTable(desc.getTableName()); 125 HMerge.merge(c, FileSystem.get(c), desc.getTableName()); 126 List<HRegionInfo> postMergeTableRegions = 127 MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(), connection, 128 desc.getTableName()); 129 LOG.info("postMergeTableRegions size=" + postMergeTableRegions.size() + 130 "; " + postMergeTableRegions); 131 assertTrue("originalTableRegions=" + originalTableRegions.size() + 132 ", postMergeTableRegions=" + postMergeTableRegions.size(), 133 postMergeTableRegions.size() < originalTableRegions.size()); 134 LOG.info("Done with merge"); 135 } finally { 136 UTIL.shutdownMiniCluster(); 137 LOG.info("After cluster shutdown"); 138 } 139 } 140 createRegion(final HTableDescriptor desc, byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)141 private HRegion createRegion(final HTableDescriptor desc, 142 byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir) 143 throws IOException { 144 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey); 145 HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration(), desc); 146 LOG.info("Created region " + region.getRegionInfo().getRegionNameAsString()); 147 for(int i = firstRow; i < firstRow + nrows; i++) { 148 Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i))); 149 put.setDurability(Durability.SKIP_WAL); 150 put.add(COLUMN_NAME, null, VALUE); 151 region.put(put); 152 if (i % 10000 == 0) { 153 LOG.info("Flushing write #" + i); 154 region.flush(true); 155 } 156 } 157 HRegion.closeHRegion(region); 158 return region; 159 } 160 setupMeta(Path rootdir, final HRegion [] regions)161 protected void setupMeta(Path rootdir, final HRegion [] regions) 162 throws IOException { 163 HRegion meta = 164 HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, rootdir, 165 UTIL.getConfiguration(), UTIL.getMetaTableDescriptor()); 166 for (HRegion r: regions) { 167 HRegion.addRegionToMETA(meta, r); 168 } 169 HRegion.closeHRegion(meta); 170 } 171 172 } 173 174