1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.test; 19 20 import java.io.File; 21 import java.util.concurrent.atomic.AtomicInteger; 22 23 import org.apache.hadoop.conf.Configuration; 24 import org.apache.hadoop.fs.FileSystem; 25 import org.apache.hadoop.fs.Path; 26 import org.apache.hadoop.fs.permission.FsPermission; 27 import org.apache.hadoop.hdfs.DFSConfigKeys; 28 import org.apache.hadoop.hdfs.MiniDFSCluster; 29 import org.junit.Test; 30 import org.junit.runners.model.FrameworkMethod; 31 import org.junit.runners.model.Statement; 32 33 public class TestHdfsHelper extends TestDirHelper { 34 35 @Override 36 @Test dummy()37 public void dummy() { 38 } 39 40 public static final String HADOOP_MINI_HDFS = "test.hadoop.hdfs"; 41 42 private static ThreadLocal<Configuration> HDFS_CONF_TL = new InheritableThreadLocal<Configuration>(); 43 44 private static ThreadLocal<Path> HDFS_TEST_DIR_TL = new InheritableThreadLocal<Path>(); 45 46 @Override apply(Statement statement, FrameworkMethod frameworkMethod, Object o)47 public Statement apply(Statement statement, FrameworkMethod frameworkMethod, Object o) { 48 TestHdfs testHdfsAnnotation = frameworkMethod.getAnnotation(TestHdfs.class); 49 if (testHdfsAnnotation != null) { 50 statement = new HdfsStatement(statement, frameworkMethod.getName()); 51 } 52 return super.apply(statement, frameworkMethod, o); 53 } 54 55 private static class HdfsStatement extends Statement { 56 private Statement statement; 57 private String testName; 58 HdfsStatement(Statement statement, String testName)59 public HdfsStatement(Statement statement, String testName) { 60 this.statement = statement; 61 this.testName = testName; 62 } 63 64 @Override evaluate()65 public void evaluate() throws Throwable { 66 MiniDFSCluster miniHdfs = null; 67 Configuration conf = HadoopUsersConfTestHelper.getBaseConf(); 68 if (Boolean.parseBoolean(System.getProperty(HADOOP_MINI_HDFS, "true"))) { 69 miniHdfs = startMiniHdfs(conf); 70 conf = miniHdfs.getConfiguration(0); 71 } 72 try { 73 HDFS_CONF_TL.set(conf); 74 HDFS_TEST_DIR_TL.set(resetHdfsTestDir(conf)); 75 statement.evaluate(); 76 } finally { 77 HDFS_CONF_TL.remove(); 78 HDFS_TEST_DIR_TL.remove(); 79 } 80 } 81 82 private static AtomicInteger counter = new AtomicInteger(); 83 resetHdfsTestDir(Configuration conf)84 private Path resetHdfsTestDir(Configuration conf) { 85 86 Path testDir = new Path("/tmp/" + testName + "-" + 87 counter.getAndIncrement()); 88 try { 89 // currentUser 90 FileSystem fs = FileSystem.get(conf); 91 fs.delete(testDir, true); 92 fs.mkdirs(testDir); 93 } catch (Exception ex) { 94 throw new RuntimeException(ex); 95 } 96 return testDir; 97 } 98 } 99 100 /** 101 * Returns the HDFS test directory for the current test, only available when the 102 * test method has been annotated with {@link TestHdfs}. 103 * 104 * @return the HDFS test directory for the current test. It is an full/absolute 105 * <code>Path</code>. 106 */ getHdfsTestDir()107 public static Path getHdfsTestDir() { 108 Path testDir = HDFS_TEST_DIR_TL.get(); 109 if (testDir == null) { 110 throw new IllegalStateException("This test does not use @TestHdfs"); 111 } 112 return testDir; 113 } 114 115 /** 116 * Returns a FileSystemAccess <code>JobConf</code> preconfigured with the FileSystemAccess cluster 117 * settings for testing. This configuration is only available when the test 118 * method has been annotated with {@link TestHdfs}. Refer to {@link HTestCase} 119 * header for details) 120 * 121 * @return the FileSystemAccess <code>JobConf</code> preconfigured with the FileSystemAccess cluster 122 * settings for testing 123 */ getHdfsConf()124 public static Configuration getHdfsConf() { 125 Configuration conf = HDFS_CONF_TL.get(); 126 if (conf == null) { 127 throw new IllegalStateException("This test does not use @TestHdfs"); 128 } 129 return new Configuration(conf); 130 } 131 132 private static MiniDFSCluster MINI_DFS = null; 133 startMiniHdfs(Configuration conf)134 private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) throws Exception { 135 if (MINI_DFS == null) { 136 if (System.getProperty("hadoop.log.dir") == null) { 137 System.setProperty("hadoop.log.dir", new File(TEST_DIR_ROOT, "hadoop-log").getAbsolutePath()); 138 } 139 if (System.getProperty("test.build.data") == null) { 140 System.setProperty("test.build.data", new File(TEST_DIR_ROOT, "hadoop-data").getAbsolutePath()); 141 } 142 143 conf = new Configuration(conf); 144 HadoopUsersConfTestHelper.addUserConf(conf); 145 conf.set("fs.hdfs.impl.disable.cache", "true"); 146 conf.set("dfs.block.access.token.enable", "false"); 147 conf.set("dfs.permissions", "true"); 148 conf.set("hadoop.security.authentication", "simple"); 149 conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); 150 conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); 151 MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); 152 builder.numDataNodes(2); 153 MiniDFSCluster miniHdfs = builder.build(); 154 FileSystem fileSystem = miniHdfs.getFileSystem(); 155 fileSystem.mkdirs(new Path("/tmp")); 156 fileSystem.mkdirs(new Path("/user")); 157 fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); 158 fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); 159 MINI_DFS = miniHdfs; 160 } 161 return MINI_DFS; 162 } 163 164 } 165