1 /**
2  * Copyright The Apache Software Foundation
3  *
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *     http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  */
20 package org.apache.hadoop.hbase.migration;
21 
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.assertFalse;
25 
26 import java.io.File;
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30 
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.FileUtil;
36 import org.apache.hadoop.fs.FsShell;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.testclassification.MediumTests;
44 import org.apache.hadoop.hbase.NamespaceDescriptor;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.Waiter;
47 import org.apache.hadoop.hbase.client.Get;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Put;
50 import org.apache.hadoop.hbase.client.Result;
51 import org.apache.hadoop.hbase.client.ResultScanner;
52 import org.apache.hadoop.hbase.client.Scan;
53 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
54 import org.apache.hadoop.hbase.regionserver.HRegion;
55 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
56 import org.apache.hadoop.hbase.security.access.AccessControlLists;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.FSTableDescriptors;
59 import org.apache.hadoop.hbase.util.FSUtils;
60 import org.apache.hadoop.util.ToolRunner;
61 import org.junit.AfterClass;
62 import org.junit.Assert;
63 import org.junit.BeforeClass;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66 
67 /**
68  * Test upgrade from no namespace in 0.94 to namespace directory structure.
69  * Mainly tests that tables are migrated and consistent. Also verifies
70  * that snapshots have been migrated correctly.
71  *
72  * <p>Uses a tarball which is an image of an 0.94 hbase.rootdir.
73  *
74  * <p>Contains tables with currentKeys as the stored keys:
75  * foo, ns1.foo, ns2.foo
76  *
77  * <p>Contains snapshots with snapshot{num}Keys as the contents:
78  * snapshot1Keys, snapshot2Keys
79  *
80  * Image also contains _acl_ table with one region and two storefiles.
81  * This is needed to test the acl table migration.
82  *
83  */
84 @Category(MediumTests.class)
85 public class TestNamespaceUpgrade {
86   static final Log LOG = LogFactory.getLog(TestNamespaceUpgrade.class);
87   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
88   private final static String snapshot1Keys[] =
89       {"1","10","2","3","4","5","6","7","8","9"};
90   private final static String snapshot2Keys[] =
91       {"1","2","3","4","5","6","7","8","9"};
92   private final static String currentKeys[] =
93       {"1","2","3","4","5","6","7","8","9","A"};
94   private final static TableName tables[] =
95     {TableName.valueOf("data"), TableName.valueOf("foo"),
96      TableName.valueOf("ns1.foo"), TableName.valueOf("ns.two.foo")};
97 
98   @BeforeClass
setUpBeforeClass()99   public static void setUpBeforeClass() throws Exception {
100     // Start up our mini cluster on top of an 0.94 root.dir that has data from
101     // a 0.94 hbase run and see if we can migrate to 0.96
102     TEST_UTIL.startMiniZKCluster();
103     TEST_UTIL.startMiniDFSCluster(1);
104     Path testdir = TEST_UTIL.getDataTestDir("TestNamespaceUpgrade");
105     // Untar our test dir.
106     File untar = untar(new File(testdir.toString()));
107     // Now copy the untar up into hdfs so when we start hbase, we'll run from it.
108     Configuration conf = TEST_UTIL.getConfiguration();
109     FsShell shell = new FsShell(conf);
110     FileSystem fs = FileSystem.get(conf);
111     // find where hbase will root itself, so we can copy filesystem there
112     Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
113     if (!fs.isDirectory(hbaseRootDir.getParent())) {
114       // mkdir at first
115       fs.mkdirs(hbaseRootDir.getParent());
116     }
117     if(org.apache.hadoop.util.VersionInfo.getVersion().startsWith("2.")) {
118       LOG.info("Hadoop version is 2.x, pre-migrating snapshot dir");
119       FileSystem localFS = FileSystem.getLocal(conf);
120       if(!localFS.rename(new Path(untar.toString(), HConstants.OLD_SNAPSHOT_DIR_NAME),
121           new Path(untar.toString(), HConstants.SNAPSHOT_DIR_NAME))) {
122         throw new IllegalStateException("Failed to move snapshot dir to 2.x expectation");
123       }
124     }
125     doFsCommand(shell,
126       new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
127     doFsCommand(shell, new String [] {"-lsr", "/"});
128     // See whats in minihdfs.
129     Configuration toolConf = TEST_UTIL.getConfiguration();
130     conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString());
131     ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"});
132     assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION));
133     doFsCommand(shell, new String [] {"-lsr", "/"});
134     TEST_UTIL.startMiniHBaseCluster(1, 1);
135 
136     for(TableName table: tables) {
137       int count = 0;
138       for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new Scan())) {
139         assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
140       }
141       Assert.assertEquals(currentKeys.length, count);
142     }
143     assertEquals(2, TEST_UTIL.getHBaseAdmin().listNamespaceDescriptors().length);
144 
145     //verify ACL table is migrated
146     HTable secureTable = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
147     ResultScanner scanner = secureTable.getScanner(new Scan());
148     int count = 0;
149     for(Result r : scanner) {
150       count++;
151     }
152     assertEquals(3, count);
153     assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(TableName.valueOf("_acl_")));
154 
155     //verify ACL table was compacted
156     List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(secureTable.getName());
157     for(HRegion region : regions) {
158       assertEquals(1, region.getStores().size());
159     }
160   }
161 
untar(final File testdir)162    static File untar(final File testdir) throws IOException {
163     // Find the src data under src/test/data
164     final String datafile = "TestNamespaceUpgrade";
165     File srcTarFile = new File(
166       System.getProperty("project.build.testSourceDirectory", "src/test") +
167       File.separator + "data" + File.separator + datafile + ".tgz");
168     File homedir = new File(testdir.toString());
169     File tgtUntarDir = new File(homedir, "hbase");
170     if (tgtUntarDir.exists()) {
171       if (!FileUtil.fullyDelete(tgtUntarDir)) {
172         throw new IOException("Failed delete of " + tgtUntarDir.toString());
173       }
174     }
175     if (!srcTarFile.exists()) {
176       throw new IOException(srcTarFile+" does not exist");
177     }
178     LOG.info("Untarring " + srcTarFile + " into " + homedir.toString());
179     FileUtil.unTar(srcTarFile, homedir);
180     Assert.assertTrue(tgtUntarDir.exists());
181     return tgtUntarDir;
182   }
183 
doFsCommand(final FsShell shell, final String [] args)184   private static void doFsCommand(final FsShell shell, final String [] args)
185   throws Exception {
186     // Run the 'put' command.
187     int errcode = shell.run(args);
188     if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode);
189   }
190 
191   @AfterClass
tearDownAfterClass()192   public static void tearDownAfterClass() throws Exception {
193     TEST_UTIL.shutdownMiniCluster();
194   }
195 
196   @Test (timeout=300000)
testSnapshots()197   public void testSnapshots() throws IOException, InterruptedException {
198     String snapshots[][] = {snapshot1Keys, snapshot2Keys};
199     for(int i = 1; i <= snapshots.length; i++) {
200       for(TableName table: tables) {
201         TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, TableName.valueOf(table+"_clone"+i));
202         FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()),
203             FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
204             LOG);
205         int count = 0;
206         for(Result res: new HTable(TEST_UTIL.getConfiguration(), table+"_clone"+i).getScanner(new
207             Scan())) {
208           assertEquals(snapshots[i-1][count++], Bytes.toString(res.getRow()));
209         }
210         Assert.assertEquals(table+"_snapshot"+i, snapshots[i-1].length, count);
211       }
212     }
213   }
214 
215   @Test (timeout=300000)
testRenameUsingSnapshots()216   public void testRenameUsingSnapshots() throws Exception {
217     String newNS = "newNS";
218     TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(newNS).build());
219     for(TableName table: tables) {
220       int count = 0;
221       for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new
222           Scan())) {
223         assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
224       }
225       TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot3", table);
226       final TableName newTableName =
227         TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
228       TEST_UTIL.getHBaseAdmin().cloneSnapshot(table + "_snapshot3", newTableName);
229       Thread.sleep(1000);
230       count = 0;
231       for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
232           Scan())) {
233         assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
234       }
235       FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath()
236           , LOG);
237       Assert.assertEquals(newTableName + "", currentKeys.length, count);
238       TEST_UTIL.getHBaseAdmin().flush(newTableName);
239       TEST_UTIL.getHBaseAdmin().majorCompact(newTableName);
240       TEST_UTIL.waitFor(30000, new Waiter.Predicate<IOException>() {
241         @Override
242         public boolean evaluate() throws IOException {
243           return TEST_UTIL.getHBaseAdmin().getCompactionState(newTableName) ==
244               AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
245         }
246       });
247     }
248 
249     String nextNS = "nextNS";
250     TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(nextNS).build());
251     for(TableName table: tables) {
252       TableName srcTable = TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
253       TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot4", srcTable);
254       TableName newTableName =
255         TableName.valueOf(nextNS + TableName.NAMESPACE_DELIM + table + "_clone4");
256       TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot4", newTableName);
257       FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(),
258         LOG);
259       int count = 0;
260       for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
261           Scan())) {
262         assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
263       }
264       Assert.assertEquals(newTableName + "", currentKeys.length, count);
265     }
266   }
267 
268   @Test (timeout=300000)
testOldDirsAreGonePostMigration()269   public void testOldDirsAreGonePostMigration() throws IOException {
270     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
271     Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
272     List <String> dirs = new ArrayList<String>(NamespaceUpgrade.NON_USER_TABLE_DIRS);
273     // Remove those that are not renamed
274     dirs.remove(HConstants.HBCK_SIDELINEDIR_NAME);
275     dirs.remove(HConstants.SNAPSHOT_DIR_NAME);
276     dirs.remove(HConstants.HBASE_TEMP_DIRECTORY);
277     for (String dir: dirs) {
278       assertFalse(fs.exists(new Path(hbaseRootDir, dir)));
279     }
280   }
281 
282   @Test (timeout=300000)
testNewDirsArePresentPostMigration()283   public void testNewDirsArePresentPostMigration() throws IOException {
284     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
285     // Below list does not include 'corrupt' because there is no 'corrupt' in the tgz
286     String [] newdirs = new String [] {HConstants.BASE_NAMESPACE_DIR,
287       HConstants.HREGION_LOGDIR_NAME};
288     Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
289     for (String dir: newdirs) {
290       assertTrue(dir, fs.exists(new Path(hbaseRootDir, dir)));
291     }
292   }
293 
294   @Test (timeout = 300000)
testACLTableMigration()295   public void testACLTableMigration() throws IOException {
296     Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testACLTable");
297     FileSystem fs = TEST_UTIL.getTestFileSystem();
298     Configuration conf = TEST_UTIL.getConfiguration();
299     byte[] FAMILY = Bytes.toBytes("l");
300     byte[] QUALIFIER = Bytes.toBytes("testUser");
301     byte[] VALUE = Bytes.toBytes("RWCA");
302 
303     // Create a Region
304     HTableDescriptor aclTable = new HTableDescriptor(TableName.valueOf("testACLTable"));
305     aclTable.addFamily(new HColumnDescriptor(FAMILY));
306     FSTableDescriptors fstd = new FSTableDescriptors(conf, fs, rootDir);
307     fstd.createTableDescriptor(aclTable);
308     HRegionInfo hriAcl = new HRegionInfo(aclTable.getTableName(), null, null);
309     HRegion region = HRegion.createHRegion(hriAcl, rootDir, conf, aclTable);
310     try {
311       // Create rows
312       Put p = new Put(Bytes.toBytes("-ROOT-"));
313       p.addImmutable(FAMILY, QUALIFIER, VALUE);
314       region.put(p);
315       p = new Put(Bytes.toBytes(".META."));
316       p.addImmutable(FAMILY, QUALIFIER, VALUE);
317       region.put(p);
318       p = new Put(Bytes.toBytes("_acl_"));
319       p.addImmutable(FAMILY, QUALIFIER, VALUE);
320       region.put(p);
321 
322       NamespaceUpgrade upgrade = new NamespaceUpgrade();
323       upgrade.updateAcls(region);
324 
325       // verify rows -ROOT- is removed
326       Get g = new Get(Bytes.toBytes("-ROOT-"));
327       Result r = region.get(g);
328       assertTrue(r == null || r.size() == 0);
329 
330       // verify rows _acl_ is renamed to hbase:acl
331       g = new Get(AccessControlLists.ACL_TABLE_NAME.toBytes());
332       r = region.get(g);
333       assertTrue(r != null && r.size() == 1);
334       assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
335 
336       // verify rows .META. is renamed to hbase:meta
337       g = new Get(TableName.META_TABLE_NAME.toBytes());
338       r = region.get(g);
339       assertTrue(r != null && r.size() == 1);
340       assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
341     } finally {
342       region.close();
343       // Delete the region
344       HRegionFileSystem.deleteRegionFromFileSystem(conf, fs,
345         FSUtils.getTableDir(rootDir, hriAcl.getTable()), hriAcl);
346     }
347   }
348 }
349