1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hdfs; 19 import static org.junit.Assert.assertTrue; 20 import static org.mockito.Mockito.doNothing; 21 import static org.mockito.Mockito.spy; 22 23 import java.io.IOException; 24 25 import org.apache.hadoop.conf.Configuration; 26 import org.apache.hadoop.fs.FSDataOutputStream; 27 import org.apache.hadoop.fs.FileSystem; 28 import org.apache.hadoop.fs.Path; 29 import org.apache.hadoop.hdfs.server.namenode.FSEditLog; 30 import org.apache.log4j.Level; 31 import org.junit.Test; 32 import org.mockito.Mockito; 33 34 public class TestRenameWhileOpen { 35 { 36 DFSTestUtil.setNameNodeLogLevel(Level.ALL); 37 } 38 39 //TODO: un-comment checkFullFile once the lease recovery is done checkFullFile(FileSystem fs, Path p)40 private static void checkFullFile(FileSystem fs, Path p) throws IOException { 41 //TestFileCreation.checkFullFile(fs, p); 42 } 43 44 /** 45 * open /user/dir1/file1 /user/dir2/file2 46 * mkdir /user/dir3 47 * move /user/dir1 /user/dir3 48 */ 49 @Test testWhileOpenRenameParent()50 public void testWhileOpenRenameParent() throws IOException { 51 Configuration conf = new HdfsConfiguration(); 52 final int MAX_IDLE_TIME = 2000; // 2s 53 conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); 54 conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); 55 conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); 56 conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 1); 57 conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, TestFileCreation.blockSize); 58 59 // create cluster 60 System.out.println("Test 1*****************************"); 61 MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); 62 FileSystem fs = null; 63 try { 64 cluster.waitActive(); 65 fs = cluster.getFileSystem(); 66 67 // Normally, the in-progress edit log would be finalized by 68 // FSEditLog#endCurrentLogSegment. For testing purposes, we 69 // disable that here. 70 FSEditLog spyLog = 71 spy(cluster.getNameNode().getFSImage().getEditLog()); 72 doNothing().when(spyLog).endCurrentLogSegment(Mockito.anyBoolean()); 73 DFSTestUtil.setEditLogForTesting(cluster.getNamesystem(), spyLog); 74 75 final int nnport = cluster.getNameNodePort(); 76 77 // create file1. 78 Path dir1 = new Path("/user/a+b/dir1"); 79 Path file1 = new Path(dir1, "file1"); 80 FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); 81 System.out.println("testFileCreationDeleteParent: " 82 + "Created file " + file1); 83 TestFileCreation.writeFile(stm1); 84 stm1.hflush(); 85 86 // create file2. 87 Path dir2 = new Path("/user/dir2"); 88 Path file2 = new Path(dir2, "file2"); 89 FSDataOutputStream stm2 = TestFileCreation.createFile(fs, file2, 1); 90 System.out.println("testFileCreationDeleteParent: " 91 + "Created file " + file2); 92 TestFileCreation.writeFile(stm2); 93 stm2.hflush(); 94 95 // move dir1 while file1 is open 96 Path dir3 = new Path("/user/dir3"); 97 fs.mkdirs(dir3); 98 fs.rename(dir1, dir3); 99 100 // create file3 101 Path file3 = new Path(dir3, "file3"); 102 FSDataOutputStream stm3 = fs.create(file3); 103 fs.rename(file3, new Path(dir3, "bozo")); 104 // Get a new block for the file. 105 TestFileCreation.writeFile(stm3, TestFileCreation.blockSize + 1); 106 stm3.hflush(); 107 108 // Stop the NameNode before closing the files. 109 // This will ensure that the write leases are still active and present 110 // in the edit log. Simiarly, there should be a pending ADD_BLOCK_OP 111 // for file3, since we just added a block to that file. 112 cluster.getNameNode().stop(); 113 114 // Restart cluster with the same namenode port as before. 115 cluster.shutdown(); 116 117 try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} 118 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 119 .format(false) 120 .build(); 121 cluster.waitActive(); 122 123 // restart cluster yet again. This triggers the code to read in 124 // persistent leases from the edit log. 125 cluster.shutdown(); 126 try {Thread.sleep(5000);} catch (InterruptedException e) {} 127 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 128 .format(false) 129 .build(); 130 cluster.waitActive(); 131 fs = cluster.getFileSystem(); 132 133 Path newfile = new Path("/user/dir3/dir1", "file1"); 134 assertTrue(!fs.exists(file1)); 135 assertTrue(fs.exists(file2)); 136 assertTrue(fs.exists(newfile)); 137 checkFullFile(fs, newfile); 138 } finally { 139 fs.close(); 140 cluster.shutdown(); 141 } 142 } 143 144 /** 145 * open /user/dir1/file1 /user/dir2/file2 146 * move /user/dir1 /user/dir3 147 */ 148 @Test testWhileOpenRenameParentToNonexistentDir()149 public void testWhileOpenRenameParentToNonexistentDir() throws IOException { 150 Configuration conf = new HdfsConfiguration(); 151 final int MAX_IDLE_TIME = 2000; // 2s 152 conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); 153 conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); 154 conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); 155 conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 1); 156 System.out.println("Test 2************************************"); 157 158 // create cluster 159 MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); 160 FileSystem fs = null; 161 try { 162 cluster.waitActive(); 163 fs = cluster.getFileSystem(); 164 final int nnport = cluster.getNameNodePort(); 165 166 // create file1. 167 Path dir1 = new Path("/user/dir1"); 168 Path file1 = new Path(dir1, "file1"); 169 FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); 170 System.out.println("testFileCreationDeleteParent: " 171 + "Created file " + file1); 172 TestFileCreation.writeFile(stm1); 173 stm1.hflush(); 174 175 // create file2. 176 Path dir2 = new Path("/user/dir2"); 177 Path file2 = new Path(dir2, "file2"); 178 FSDataOutputStream stm2 = TestFileCreation.createFile(fs, file2, 1); 179 System.out.println("testFileCreationDeleteParent: " 180 + "Created file " + file2); 181 TestFileCreation.writeFile(stm2); 182 stm2.hflush(); 183 184 // move dir1 while file1 is open 185 Path dir3 = new Path("/user/dir3"); 186 fs.rename(dir1, dir3); 187 188 // restart cluster with the same namenode port as before. 189 // This ensures that leases are persisted in fsimage. 190 cluster.shutdown(); 191 try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} 192 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 193 .format(false) 194 .build(); 195 cluster.waitActive(); 196 197 // restart cluster yet again. This triggers the code to read in 198 // persistent leases from fsimage. 199 cluster.shutdown(); 200 try {Thread.sleep(5000);} catch (InterruptedException e) {} 201 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 202 .format(false) 203 .build(); 204 cluster.waitActive(); 205 fs = cluster.getFileSystem(); 206 207 Path newfile = new Path("/user/dir3", "file1"); 208 assertTrue(!fs.exists(file1)); 209 assertTrue(fs.exists(file2)); 210 assertTrue(fs.exists(newfile)); 211 checkFullFile(fs, newfile); 212 } finally { 213 fs.close(); 214 cluster.shutdown(); 215 } 216 } 217 218 /** 219 * open /user/dir1/file1 220 * mkdir /user/dir2 221 * move /user/dir1/file1 /user/dir2/ 222 */ 223 @Test testWhileOpenRenameToExistentDirectory()224 public void testWhileOpenRenameToExistentDirectory() throws IOException { 225 Configuration conf = new HdfsConfiguration(); 226 final int MAX_IDLE_TIME = 2000; // 2s 227 conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); 228 conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); 229 conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); 230 conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 1); 231 System.out.println("Test 3************************************"); 232 233 // create cluster 234 MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); 235 FileSystem fs = null; 236 try { 237 cluster.waitActive(); 238 fs = cluster.getFileSystem(); 239 final int nnport = cluster.getNameNodePort(); 240 241 // create file1. 242 Path dir1 = new Path("/user/dir1"); 243 Path file1 = new Path(dir1, "file1"); 244 FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); 245 System.out.println("testFileCreationDeleteParent: " + 246 "Created file " + file1); 247 TestFileCreation.writeFile(stm1); 248 stm1.hflush(); 249 250 Path dir2 = new Path("/user/dir2"); 251 fs.mkdirs(dir2); 252 253 fs.rename(file1, dir2); 254 255 // restart cluster with the same namenode port as before. 256 // This ensures that leases are persisted in fsimage. 257 cluster.shutdown(); 258 try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} 259 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 260 .format(false) 261 .build(); 262 cluster.waitActive(); 263 264 // restart cluster yet again. This triggers the code to read in 265 // persistent leases from fsimage. 266 cluster.shutdown(); 267 try {Thread.sleep(5000);} catch (InterruptedException e) {} 268 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 269 .format(false) 270 .build(); 271 cluster.waitActive(); 272 fs = cluster.getFileSystem(); 273 274 Path newfile = new Path("/user/dir2", "file1"); 275 assertTrue(!fs.exists(file1)); 276 assertTrue(fs.exists(newfile)); 277 checkFullFile(fs, newfile); 278 } finally { 279 fs.close(); 280 cluster.shutdown(); 281 } 282 } 283 284 /** 285 * open /user/dir1/file1 286 * move /user/dir1/file1 /user/dir2/ 287 */ 288 @Test testWhileOpenRenameToNonExistentDirectory()289 public void testWhileOpenRenameToNonExistentDirectory() throws IOException { 290 Configuration conf = new HdfsConfiguration(); 291 final int MAX_IDLE_TIME = 2000; // 2s 292 conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); 293 conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); 294 conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); 295 conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 1); 296 System.out.println("Test 4************************************"); 297 298 // create cluster 299 MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); 300 FileSystem fs = null; 301 try { 302 cluster.waitActive(); 303 fs = cluster.getFileSystem(); 304 final int nnport = cluster.getNameNodePort(); 305 306 // create file1. 307 Path dir1 = new Path("/user/dir1"); 308 Path file1 = new Path(dir1, "file1"); 309 FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); 310 System.out.println("testFileCreationDeleteParent: " 311 + "Created file " + file1); 312 TestFileCreation.writeFile(stm1); 313 stm1.hflush(); 314 315 Path dir2 = new Path("/user/dir2"); 316 317 fs.rename(file1, dir2); 318 319 // restart cluster with the same namenode port as before. 320 // This ensures that leases are persisted in fsimage. 321 cluster.shutdown(); 322 try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} 323 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 324 .format(false) 325 .build(); 326 cluster.waitActive(); 327 328 // restart cluster yet again. This triggers the code to read in 329 // persistent leases from fsimage. 330 cluster.shutdown(); 331 try {Thread.sleep(5000);} catch (InterruptedException e) {} 332 cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport) 333 .format(false) 334 .build(); 335 cluster.waitActive(); 336 fs = cluster.getFileSystem(); 337 338 Path newfile = new Path("/user", "dir2"); 339 assertTrue(!fs.exists(file1)); 340 assertTrue(fs.exists(newfile)); 341 checkFullFile(fs, newfile); 342 } finally { 343 fs.close(); 344 cluster.shutdown(); 345 } 346 } 347 } 348