1 /**
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 package org.apache.hadoop.test;
19 
20 import java.io.File;
21 import java.util.concurrent.atomic.AtomicInteger;
22 
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.fs.FileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.fs.permission.FsPermission;
27 import org.apache.hadoop.hdfs.DFSConfigKeys;
28 import org.apache.hadoop.hdfs.MiniDFSCluster;
29 import org.junit.Test;
30 import org.junit.runners.model.FrameworkMethod;
31 import org.junit.runners.model.Statement;
32 
33 public class TestHdfsHelper extends TestDirHelper {
34 
35   @Override
36   @Test
dummy()37   public void dummy() {
38   }
39 
40   public static final String HADOOP_MINI_HDFS = "test.hadoop.hdfs";
41 
42   private static ThreadLocal<Configuration> HDFS_CONF_TL = new InheritableThreadLocal<Configuration>();
43 
44   private static ThreadLocal<Path> HDFS_TEST_DIR_TL = new InheritableThreadLocal<Path>();
45 
46   @Override
apply(Statement statement, FrameworkMethod frameworkMethod, Object o)47   public Statement apply(Statement statement, FrameworkMethod frameworkMethod, Object o) {
48     TestHdfs testHdfsAnnotation = frameworkMethod.getAnnotation(TestHdfs.class);
49     if (testHdfsAnnotation != null) {
50       statement = new HdfsStatement(statement, frameworkMethod.getName());
51     }
52     return super.apply(statement, frameworkMethod, o);
53   }
54 
55   private static class HdfsStatement extends Statement {
56     private Statement statement;
57     private String testName;
58 
HdfsStatement(Statement statement, String testName)59     public HdfsStatement(Statement statement, String testName) {
60       this.statement = statement;
61       this.testName = testName;
62     }
63 
64     @Override
evaluate()65     public void evaluate() throws Throwable {
66       MiniDFSCluster miniHdfs = null;
67       Configuration conf = HadoopUsersConfTestHelper.getBaseConf();
68       if (Boolean.parseBoolean(System.getProperty(HADOOP_MINI_HDFS, "true"))) {
69         miniHdfs = startMiniHdfs(conf);
70         conf = miniHdfs.getConfiguration(0);
71       }
72       try {
73         HDFS_CONF_TL.set(conf);
74         HDFS_TEST_DIR_TL.set(resetHdfsTestDir(conf));
75         statement.evaluate();
76       } finally {
77         HDFS_CONF_TL.remove();
78         HDFS_TEST_DIR_TL.remove();
79       }
80     }
81 
82     private static AtomicInteger counter = new AtomicInteger();
83 
resetHdfsTestDir(Configuration conf)84     private Path resetHdfsTestDir(Configuration conf) {
85 
86       Path testDir = new Path("/tmp/" + testName + "-" +
87         counter.getAndIncrement());
88       try {
89         // currentUser
90         FileSystem fs = FileSystem.get(conf);
91         fs.delete(testDir, true);
92         fs.mkdirs(testDir);
93       } catch (Exception ex) {
94         throw new RuntimeException(ex);
95       }
96       return testDir;
97     }
98   }
99 
100   /**
101    * Returns the HDFS test directory for the current test, only available when the
102    * test method has been annotated with {@link TestHdfs}.
103    *
104    * @return the HDFS test directory for the current test. It is an full/absolute
105    *         <code>Path</code>.
106    */
getHdfsTestDir()107   public static Path getHdfsTestDir() {
108     Path testDir = HDFS_TEST_DIR_TL.get();
109     if (testDir == null) {
110       throw new IllegalStateException("This test does not use @TestHdfs");
111     }
112     return testDir;
113   }
114 
115   /**
116    * Returns a FileSystemAccess <code>JobConf</code> preconfigured with the FileSystemAccess cluster
117    * settings for testing. This configuration is only available when the test
118    * method has been annotated with {@link TestHdfs}. Refer to {@link HTestCase}
119    * header for details)
120    *
121    * @return the FileSystemAccess <code>JobConf</code> preconfigured with the FileSystemAccess cluster
122    *         settings for testing
123    */
getHdfsConf()124   public static Configuration getHdfsConf() {
125     Configuration conf = HDFS_CONF_TL.get();
126     if (conf == null) {
127       throw new IllegalStateException("This test does not use @TestHdfs");
128     }
129     return new Configuration(conf);
130   }
131 
132   private static MiniDFSCluster MINI_DFS = null;
133 
startMiniHdfs(Configuration conf)134   private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) throws Exception {
135     if (MINI_DFS == null) {
136       if (System.getProperty("hadoop.log.dir") == null) {
137         System.setProperty("hadoop.log.dir", new File(TEST_DIR_ROOT, "hadoop-log").getAbsolutePath());
138       }
139       if (System.getProperty("test.build.data") == null) {
140         System.setProperty("test.build.data", new File(TEST_DIR_ROOT, "hadoop-data").getAbsolutePath());
141       }
142 
143       conf = new Configuration(conf);
144       HadoopUsersConfTestHelper.addUserConf(conf);
145       conf.set("fs.hdfs.impl.disable.cache", "true");
146       conf.set("dfs.block.access.token.enable", "false");
147       conf.set("dfs.permissions", "true");
148       conf.set("hadoop.security.authentication", "simple");
149       conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
150       conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
151       MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
152       builder.numDataNodes(2);
153       MiniDFSCluster miniHdfs = builder.build();
154       FileSystem fileSystem = miniHdfs.getFileSystem();
155       fileSystem.mkdirs(new Path("/tmp"));
156       fileSystem.mkdirs(new Path("/user"));
157       fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx"));
158       fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx"));
159       MINI_DFS = miniHdfs;
160     }
161     return MINI_DFS;
162   }
163 
164 }
165