1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.master.cleaner; 19 20 import static org.junit.Assert.assertEquals; 21 import static org.junit.Assert.assertFalse; 22 import static org.junit.Assert.assertTrue; 23 24 import java.io.IOException; 25 26 import org.apache.hadoop.conf.Configuration; 27 import org.apache.hadoop.fs.FileStatus; 28 import org.apache.hadoop.fs.FileSystem; 29 import org.apache.hadoop.fs.Path; 30 import org.apache.hadoop.hbase.ChoreService; 31 import org.apache.hadoop.hbase.CoordinatedStateManager; 32 import org.apache.hadoop.hbase.HBaseTestingUtility; 33 import org.apache.hadoop.hbase.HRegionInfo; 34 import org.apache.hadoop.hbase.Server; 35 import org.apache.hadoop.hbase.ServerName; 36 import org.apache.hadoop.hbase.TableName; 37 import org.apache.hadoop.hbase.client.ClusterConnection; 38 import org.apache.hadoop.hbase.io.HFileLink; 39 import org.apache.hadoop.hbase.testclassification.SmallTests; 40 import org.apache.hadoop.hbase.util.FSUtils; 41 import org.apache.hadoop.hbase.util.HFileArchiveUtil; 42 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 43 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; 44 import org.junit.Test; 45 import org.junit.experimental.categories.Category; 46 47 /** 48 * Test the HFileLink Cleaner. 49 * HFiles with links cannot be deleted until a link is present. 50 */ 51 @Category(SmallTests.class) 52 public class TestHFileLinkCleaner { 53 54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 55 56 @Test testHFileLinkCleaning()57 public void testHFileLinkCleaning() throws Exception { 58 Configuration conf = TEST_UTIL.getConfiguration(); 59 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); 60 conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); 61 Path rootDir = FSUtils.getRootDir(conf); 62 FileSystem fs = FileSystem.get(conf); 63 64 final TableName tableName = TableName.valueOf("test-table"); 65 final TableName tableLinkName = TableName.valueOf("test-link"); 66 final String hfileName = "1234567890"; 67 final String familyName = "cf"; 68 69 HRegionInfo hri = new HRegionInfo(tableName); 70 HRegionInfo hriLink = new HRegionInfo(tableLinkName); 71 72 Path archiveDir = HFileArchiveUtil.getArchivePath(conf); 73 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, 74 tableName, hri.getEncodedName(), familyName); 75 Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, 76 tableLinkName, hriLink.getEncodedName(), familyName); 77 78 // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); 79 Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); 80 fs.mkdirs(familyPath); 81 Path hfilePath = new Path(familyPath, hfileName); 82 fs.createNewFile(hfilePath); 83 84 // Create link to hfile 85 Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, 86 hriLink.getEncodedName(), familyName); 87 fs.mkdirs(familyLinkPath); 88 HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); 89 Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); 90 assertTrue(fs.exists(linkBackRefDir)); 91 FileStatus[] backRefs = fs.listStatus(linkBackRefDir); 92 assertEquals(1, backRefs.length); 93 Path linkBackRef = backRefs[0].getPath(); 94 95 // Initialize cleaner 96 final long ttl = 1000; 97 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); 98 Server server = new DummyServer(); 99 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir); 100 101 // Link backref cannot be removed 102 cleaner.chore(); 103 assertTrue(fs.exists(linkBackRef)); 104 assertTrue(fs.exists(hfilePath)); 105 106 // Link backref can be removed 107 fs.rename(FSUtils.getTableDir(rootDir, tableLinkName), 108 FSUtils.getTableDir(archiveDir, tableLinkName)); 109 cleaner.chore(); 110 assertFalse("Link should be deleted", fs.exists(linkBackRef)); 111 112 // HFile can be removed 113 Thread.sleep(ttl * 2); 114 cleaner.chore(); 115 assertFalse("HFile should be deleted", fs.exists(hfilePath)); 116 117 // Remove everything 118 for (int i = 0; i < 4; ++i) { 119 Thread.sleep(ttl * 2); 120 cleaner.chore(); 121 } 122 assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); 123 assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); 124 } 125 getFamilyDirPath(final Path rootDir, final TableName table, final String region, final String family)126 private static Path getFamilyDirPath (final Path rootDir, final TableName table, 127 final String region, final String family) { 128 return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family); 129 } 130 131 static class DummyServer implements Server { 132 133 @Override getConfiguration()134 public Configuration getConfiguration() { 135 return TEST_UTIL.getConfiguration(); 136 } 137 138 @Override getZooKeeper()139 public ZooKeeperWatcher getZooKeeper() { 140 try { 141 return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); 142 } catch (IOException e) { 143 e.printStackTrace(); 144 } 145 return null; 146 } 147 148 @Override getCoordinatedStateManager()149 public CoordinatedStateManager getCoordinatedStateManager() { 150 return null; 151 } 152 153 @Override getConnection()154 public ClusterConnection getConnection() { 155 return null; 156 } 157 158 @Override getMetaTableLocator()159 public MetaTableLocator getMetaTableLocator() { 160 return null; 161 } 162 163 @Override getServerName()164 public ServerName getServerName() { 165 return ServerName.valueOf("regionserver,60020,000000"); 166 } 167 168 @Override abort(String why, Throwable e)169 public void abort(String why, Throwable e) {} 170 171 @Override isAborted()172 public boolean isAborted() { 173 return false; 174 } 175 176 @Override stop(String why)177 public void stop(String why) {} 178 179 @Override isStopped()180 public boolean isStopped() { 181 return false; 182 } 183 184 @Override getChoreService()185 public ChoreService getChoreService() { 186 return null; 187 } 188 } 189 } 190