1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hdfs; 19 20 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType.DATA_NODE; 21 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType.NAME_NODE; 22 import static org.junit.Assert.*; 23 24 import java.io.File; 25 import java.io.IOException; 26 import java.util.Collections; 27 import java.util.List; 28 29 import org.apache.commons.logging.Log; 30 import org.apache.commons.logging.LogFactory; 31 import org.apache.hadoop.conf.Configuration; 32 import org.apache.hadoop.fs.Path; 33 import org.apache.hadoop.hdfs.protocol.HdfsConstants; 34 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; 35 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; 36 import org.apache.hadoop.hdfs.server.common.StorageInfo; 37 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; 38 import org.apache.hadoop.hdfs.server.namenode.NameNode; 39 import org.apache.hadoop.util.StringUtils; 40 import org.junit.After; 41 import org.junit.Test; 42 43 import com.google.common.base.Charsets; 44 import com.google.common.collect.Lists; 45 46 /** 47 * This test ensures the appropriate response (successful or failure) from 48 * the system when the system is rolled back under various storage state and 49 * version conditions. 50 */ 51 public class TestDFSRollback { 52 53 private static final Log LOG = LogFactory.getLog( 54 "org.apache.hadoop.hdfs.TestDFSRollback"); 55 private Configuration conf; 56 private int testCounter = 0; 57 private MiniDFSCluster cluster = null; 58 59 /** 60 * Writes an INFO log message containing the parameters. 61 */ log(String label, int numDirs)62 void log(String label, int numDirs) { 63 LOG.info("============================================================"); 64 LOG.info("***TEST " + (testCounter++) + "*** " 65 + label + ":" 66 + " numDirs="+numDirs); 67 } 68 69 /** 70 * Verify that the new current directory is the old previous. 71 * It is assumed that the server has recovered and rolled back. 72 */ checkResult(NodeType nodeType, String[] baseDirs)73 void checkResult(NodeType nodeType, String[] baseDirs) throws Exception { 74 List<File> curDirs = Lists.newArrayList(); 75 for (String baseDir : baseDirs) { 76 File curDir = new File(baseDir, "current"); 77 curDirs.add(curDir); 78 switch (nodeType) { 79 case NAME_NODE: 80 FSImageTestUtil.assertReasonableNameCurrentDir(curDir); 81 break; 82 case DATA_NODE: 83 assertEquals( 84 UpgradeUtilities.checksumContents(nodeType, curDir, false), 85 UpgradeUtilities.checksumMasterDataNodeContents()); 86 break; 87 } 88 } 89 90 FSImageTestUtil.assertParallelFilesAreIdentical( 91 curDirs, Collections.<String>emptySet()); 92 93 for (int i = 0; i < baseDirs.length; i++) { 94 assertFalse(new File(baseDirs[i],"previous").isDirectory()); 95 } 96 } 97 98 /** 99 * Attempts to start a NameNode with the given operation. Starting 100 * the NameNode should throw an exception. 101 */ startNameNodeShouldFail(String searchString)102 void startNameNodeShouldFail(String searchString) { 103 try { 104 NameNode.doRollback(conf, false); 105 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 106 .format(false) 107 .manageDataDfsDirs(false) 108 .manageNameDfsDirs(false) 109 .build(); // should fail 110 throw new AssertionError("NameNode should have failed to start"); 111 } catch (Exception expected) { 112 if (!expected.getMessage().contains(searchString)) { 113 fail("Expected substring '" + searchString + "' in exception " + 114 "but got: " + StringUtils.stringifyException(expected)); 115 } 116 // expected 117 } 118 } 119 120 /** 121 * Attempts to start a DataNode with the given operation. Starting 122 * the given block pool should fail. 123 * @param operation startup option 124 * @param bpid block pool Id that should fail to start 125 * @throws IOException 126 */ startBlockPoolShouldFail(StartupOption operation, String bpid)127 void startBlockPoolShouldFail(StartupOption operation, String bpid) 128 throws IOException { 129 cluster.startDataNodes(conf, 1, false, operation, null); // should fail 130 assertFalse("Block pool " + bpid + " should have failed to start", 131 cluster.getDataNodes().get(0).isBPServiceAlive(bpid)); 132 } 133 134 /** 135 * This test attempts to rollback the NameNode and DataNode under 136 * a number of valid and invalid conditions. 137 */ 138 @Test testRollback()139 public void testRollback() throws Exception { 140 File[] baseDirs; 141 UpgradeUtilities.initialize(); 142 143 StorageInfo storageInfo = null; 144 for (int numDirs = 1; numDirs <= 2; numDirs++) { 145 conf = new HdfsConfiguration(); 146 conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); 147 conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf); 148 String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY); 149 String[] dataNodeDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); 150 151 log("Normal NameNode rollback", numDirs); 152 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 153 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 154 NameNode.doRollback(conf, false); 155 checkResult(NAME_NODE, nameNodeDirs); 156 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 157 158 log("Normal DataNode rollback", numDirs); 159 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 160 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 161 NameNode.doRollback(conf, false); 162 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 163 .format(false) 164 .manageDataDfsDirs(false) 165 .manageNameDfsDirs(false) 166 .dnStartupOption(StartupOption.ROLLBACK) 167 .build(); 168 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); 169 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "previous"); 170 cluster.startDataNodes(conf, 1, false, StartupOption.ROLLBACK, null); 171 checkResult(DATA_NODE, dataNodeDirs); 172 cluster.shutdown(); 173 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 174 UpgradeUtilities.createEmptyDirs(dataNodeDirs); 175 176 log("Normal BlockPool rollback", numDirs); 177 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 178 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 179 NameNode.doRollback(conf, false); 180 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 181 .format(false) 182 .manageDataDfsDirs(false) 183 .manageNameDfsDirs(false) 184 .dnStartupOption(StartupOption.ROLLBACK) 185 .build(); 186 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); 187 UpgradeUtilities.createBlockPoolStorageDirs(dataNodeDirs, "current", 188 UpgradeUtilities.getCurrentBlockPoolID(cluster)); 189 // Create a previous snapshot for the blockpool 190 UpgradeUtilities.createBlockPoolStorageDirs(dataNodeDirs, "previous", 191 UpgradeUtilities.getCurrentBlockPoolID(cluster)); 192 // Put newer layout version in current. 193 storageInfo = new StorageInfo( 194 HdfsConstants.DATANODE_LAYOUT_VERSION - 1, 195 UpgradeUtilities.getCurrentNamespaceID(cluster), 196 UpgradeUtilities.getCurrentClusterID(cluster), 197 UpgradeUtilities.getCurrentFsscTime(cluster), 198 NodeType.DATA_NODE); 199 200 // Overwrite VERSION file in the current directory of 201 // volume directories and block pool slice directories 202 // with a layout version from future. 203 File[] dataCurrentDirs = new File[dataNodeDirs.length]; 204 for (int i=0; i<dataNodeDirs.length; i++) { 205 dataCurrentDirs[i] = new File((new Path(dataNodeDirs[i] 206 + "/current")).toString()); 207 } 208 UpgradeUtilities.createDataNodeVersionFile( 209 dataCurrentDirs, 210 storageInfo, 211 UpgradeUtilities.getCurrentBlockPoolID(cluster)); 212 213 cluster.startDataNodes(conf, 1, false, StartupOption.ROLLBACK, null); 214 assertTrue(cluster.isDataNodeUp()); 215 216 cluster.shutdown(); 217 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 218 UpgradeUtilities.createEmptyDirs(dataNodeDirs); 219 220 log("NameNode rollback without existing previous dir", numDirs); 221 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 222 startNameNodeShouldFail( 223 "None of the storage directories contain previous fs state"); 224 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 225 226 log("DataNode rollback without existing previous dir", numDirs); 227 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 228 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 229 .format(false) 230 .manageDataDfsDirs(false) 231 .manageNameDfsDirs(false) 232 .startupOption(StartupOption.UPGRADE) 233 .build(); 234 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); 235 cluster.startDataNodes(conf, 1, false, StartupOption.ROLLBACK, null); 236 cluster.shutdown(); 237 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 238 UpgradeUtilities.createEmptyDirs(dataNodeDirs); 239 240 log("DataNode rollback with future stored layout version in previous", numDirs); 241 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 242 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 243 NameNode.doRollback(conf, false); 244 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 245 .format(false) 246 .manageDataDfsDirs(false) 247 .manageNameDfsDirs(false) 248 .dnStartupOption(StartupOption.ROLLBACK) 249 .build(); 250 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); 251 baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "previous"); 252 storageInfo = new StorageInfo(Integer.MIN_VALUE, 253 UpgradeUtilities.getCurrentNamespaceID(cluster), 254 UpgradeUtilities.getCurrentClusterID(cluster), 255 UpgradeUtilities.getCurrentFsscTime(cluster), 256 NodeType.DATA_NODE); 257 258 UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo, 259 UpgradeUtilities.getCurrentBlockPoolID(cluster)); 260 261 startBlockPoolShouldFail(StartupOption.ROLLBACK, 262 cluster.getNamesystem().getBlockPoolId()); 263 cluster.shutdown(); 264 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 265 UpgradeUtilities.createEmptyDirs(dataNodeDirs); 266 267 log("DataNode rollback with newer fsscTime in previous", numDirs); 268 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 269 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 270 NameNode.doRollback(conf, false); 271 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) 272 .format(false) 273 .manageDataDfsDirs(false) 274 .manageNameDfsDirs(false) 275 .dnStartupOption(StartupOption.ROLLBACK) 276 .build(); 277 278 UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); 279 baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "previous"); 280 storageInfo = new StorageInfo(HdfsConstants.DATANODE_LAYOUT_VERSION, 281 UpgradeUtilities.getCurrentNamespaceID(cluster), 282 UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE, 283 NodeType.DATA_NODE); 284 285 UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo, 286 UpgradeUtilities.getCurrentBlockPoolID(cluster)); 287 288 startBlockPoolShouldFail(StartupOption.ROLLBACK, 289 cluster.getNamesystem().getBlockPoolId()); 290 cluster.shutdown(); 291 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 292 UpgradeUtilities.createEmptyDirs(dataNodeDirs); 293 294 log("NameNode rollback with no edits file", numDirs); 295 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 296 baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 297 deleteMatchingFiles(baseDirs, "edits.*"); 298 startNameNodeShouldFail("Gap in transactions"); 299 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 300 301 log("NameNode rollback with no image file", numDirs); 302 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 303 baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 304 deleteMatchingFiles(baseDirs, "fsimage_.*"); 305 startNameNodeShouldFail("No valid image files found"); 306 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 307 308 log("NameNode rollback with corrupt version file", numDirs); 309 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 310 baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 311 for (File f : baseDirs) { 312 UpgradeUtilities.corruptFile( 313 new File(f,"VERSION"), 314 "layoutVersion".getBytes(Charsets.UTF_8), 315 "xxxxxxxxxxxxx".getBytes(Charsets.UTF_8)); 316 } 317 startNameNodeShouldFail("file VERSION has layoutVersion missing"); 318 319 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 320 321 log("NameNode rollback with old layout version in previous", numDirs); 322 UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); 323 baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); 324 storageInfo = new StorageInfo(1, 325 UpgradeUtilities.getCurrentNamespaceID(null), 326 UpgradeUtilities.getCurrentClusterID(null), 327 UpgradeUtilities.getCurrentFsscTime(null), NodeType.NAME_NODE); 328 329 UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs, 330 storageInfo, UpgradeUtilities.getCurrentBlockPoolID(cluster)); 331 startNameNodeShouldFail("Cannot rollback to storage version 1 using this version"); 332 UpgradeUtilities.createEmptyDirs(nameNodeDirs); 333 } // end numDir loop 334 } 335 deleteMatchingFiles(File[] baseDirs, String regex)336 private void deleteMatchingFiles(File[] baseDirs, String regex) { 337 for (File baseDir : baseDirs) { 338 for (File f : baseDir.listFiles()) { 339 if (f.getName().matches(regex)) { 340 f.delete(); 341 } 342 } 343 } 344 } 345 346 @After tearDown()347 public void tearDown() throws Exception { 348 LOG.info("Shutting down MiniDFSCluster"); 349 if (cluster != null) cluster.shutdown(); 350 } 351 main(String[] args)352 public static void main(String[] args) throws Exception { 353 new TestDFSRollback().testRollback(); 354 } 355 356 } 357 358 359