1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.tools; 20 21 import java.util.Map; 22 23 import org.apache.hadoop.conf.Configuration; 24 import org.apache.hadoop.fs.FileSystem; 25 import org.apache.hadoop.fs.Path; 26 import org.apache.hadoop.hdfs.DFSConfigKeys; 27 import org.apache.hadoop.hdfs.MiniDFSCluster; 28 import org.apache.hadoop.io.IOUtils; 29 import org.apache.hadoop.tools.util.DistCpTestUtils; 30 31 import org.junit.AfterClass; 32 import org.junit.BeforeClass; 33 import org.junit.Test; 34 35 import com.google.common.collect.Maps; 36 37 /** 38 * Tests distcp in combination with HDFS raw.* XAttrs. 39 */ 40 public class TestDistCpWithRawXAttrs { 41 42 private static MiniDFSCluster cluster; 43 private static Configuration conf; 44 private static FileSystem fs; 45 46 private static final String rawName1 = "raw.a1"; 47 private static final byte[] rawValue1 = {0x37, 0x38, 0x39}; 48 private static final String userName1 = "user.a1"; 49 private static final byte[] userValue1 = {0x38, 0x38, 0x38}; 50 51 private static final Path dir1 = new Path("/src/dir1"); 52 private static final Path subDir1 = new Path(dir1, "subdir1"); 53 private static final Path file1 = new Path("/src/file1"); 54 private static final String rawRootName = "/.reserved/raw"; 55 private static final String rootedDestName = "/dest"; 56 private static final String rootedSrcName = "/src"; 57 private static final String rawDestName = "/.reserved/raw/dest"; 58 private static final String rawSrcName = "/.reserved/raw/src"; 59 60 @BeforeClass init()61 public static void init() throws Exception { 62 conf = new Configuration(); 63 conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); 64 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true) 65 .build(); 66 cluster.waitActive(); 67 fs = cluster.getFileSystem(); 68 } 69 70 @AfterClass shutdown()71 public static void shutdown() { 72 IOUtils.cleanup(null, fs); 73 if (cluster != null) { 74 cluster.shutdown(); 75 } 76 } 77 78 /* Test that XAttrs and raw.* XAttrs are preserved when appropriate. */ 79 @Test testPreserveRawXAttrs1()80 public void testPreserveRawXAttrs1() throws Exception { 81 final String relSrc = "/./.reserved/../.reserved/raw/../raw/src/../src"; 82 final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest"; 83 doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true, 84 DistCpConstants.SUCCESS); 85 doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px", 86 false, true, DistCpConstants.SUCCESS); 87 doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px", 88 false, true, DistCpConstants.INVALID_ARGUMENT); 89 doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px", 90 false, true, DistCpConstants.INVALID_ARGUMENT); 91 doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px", 92 true, true, DistCpConstants.SUCCESS); 93 final Path savedWd = fs.getWorkingDirectory(); 94 try { 95 fs.setWorkingDirectory(new Path("/.reserved/raw")); 96 doTestPreserveRawXAttrs("../.." + rawSrcName, "../.." + rawDestName, 97 "-px", true, true, DistCpConstants.SUCCESS); 98 } finally { 99 fs.setWorkingDirectory(savedWd); 100 } 101 } 102 103 /* Test that XAttrs are not preserved and raw.* are when appropriate. */ 104 @Test testPreserveRawXAttrs2()105 public void testPreserveRawXAttrs2() throws Exception { 106 doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p", 107 false, false, DistCpConstants.SUCCESS); 108 doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p", 109 false, false, DistCpConstants.INVALID_ARGUMENT); 110 doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p", 111 false, false, DistCpConstants.INVALID_ARGUMENT); 112 doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p", 113 true, false, DistCpConstants.SUCCESS); 114 } 115 116 /* Test that XAttrs are not preserved and raw.* are when appropriate. */ 117 @Test testPreserveRawXAttrs3()118 public void testPreserveRawXAttrs3() throws Exception { 119 doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null, 120 false, false, DistCpConstants.SUCCESS); 121 doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null, 122 false, false, DistCpConstants.INVALID_ARGUMENT); 123 doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null, 124 false, false, DistCpConstants.INVALID_ARGUMENT); 125 doTestPreserveRawXAttrs(rawSrcName, rawDestName, null, 126 true, false, DistCpConstants.SUCCESS); 127 } 128 129 private static Path[] pathnames = { new Path("dir1"), 130 new Path("dir1/subdir1"), 131 new Path("file1") }; 132 makeFilesAndDirs(FileSystem fs)133 private static void makeFilesAndDirs(FileSystem fs) throws Exception { 134 fs.delete(new Path("/src"), true); 135 fs.delete(new Path("/dest"), true); 136 fs.mkdirs(subDir1); 137 fs.create(file1).close(); 138 } 139 initXAttrs()140 private void initXAttrs() throws Exception { 141 makeFilesAndDirs(fs); 142 for (Path p : pathnames) { 143 fs.setXAttr(new Path(rawRootName + "/src", p), rawName1, rawValue1); 144 fs.setXAttr(new Path(rawRootName + "/src", p), userName1, userValue1); 145 } 146 } 147 doTestPreserveRawXAttrs(String src, String dest, String preserveOpts, boolean expectRaw, boolean expectUser, int expectedExitCode)148 private void doTestPreserveRawXAttrs(String src, String dest, 149 String preserveOpts, boolean expectRaw, boolean expectUser, 150 int expectedExitCode) throws Exception { 151 initXAttrs(); 152 153 DistCpTestUtils.assertRunDistCp(expectedExitCode, src, dest, 154 preserveOpts, conf); 155 156 if (expectedExitCode == DistCpConstants.SUCCESS) { 157 Map<String, byte[]> xAttrs = Maps.newHashMap(); 158 for (Path p : pathnames) { 159 xAttrs.clear(); 160 if (expectRaw) { 161 xAttrs.put(rawName1, rawValue1); 162 } 163 if (expectUser) { 164 xAttrs.put(userName1, userValue1); 165 } 166 DistCpTestUtils.assertXAttrs(new Path(dest, p), fs, xAttrs); 167 } 168 } 169 } 170 } 171