1 /*- 2 * See the file LICENSE for redistribution information. 3 * 4 * Copyright (c) 2002, 2014 Oracle and/or its affiliates. All rights reserved. 5 * 6 */ 7 8 package com.sleepycat.je.recovery; 9 10 import static org.junit.Assert.assertEquals; 11 import static org.junit.Assert.assertTrue; 12 import static org.junit.Assert.fail; 13 14 import java.io.File; 15 import java.io.IOException; 16 import java.io.RandomAccessFile; 17 import java.util.HashMap; 18 import java.util.List; 19 import java.util.Map; 20 import java.util.Set; 21 22 import org.junit.Test; 23 24 import com.sleepycat.je.Database; 25 import com.sleepycat.je.DatabaseConfig; 26 import com.sleepycat.je.DatabaseEntry; 27 import com.sleepycat.je.DbInternal; 28 import com.sleepycat.je.Environment; 29 import com.sleepycat.je.EnvironmentConfig; 30 import com.sleepycat.je.Transaction; 31 import com.sleepycat.je.config.EnvironmentParams; 32 import com.sleepycat.je.dbi.DatabaseImpl; 33 import com.sleepycat.je.dbi.EnvironmentImpl; 34 import com.sleepycat.je.dbi.NodeSequence; 35 import com.sleepycat.je.log.FileManager; 36 import com.sleepycat.je.log.LogEntryType; 37 import com.sleepycat.je.log.SearchFileReader; 38 import com.sleepycat.je.tree.IN; 39 import com.sleepycat.je.util.StringDbt; 40 import com.sleepycat.je.util.TestUtils; 41 import com.sleepycat.je.utilint.DbLsn; 42 43 public class RecoveryEdgeTest extends RecoveryTestBase { 44 45 @Test testNoLogFiles()46 public void testNoLogFiles() 47 throws Throwable { 48 49 /* Creating an environment runs recovery. */ 50 Environment env = null; 51 try { 52 EnvironmentConfig noFileConfig = TestUtils.initEnvConfig(); 53 /* Don't checkpoint utilization info for this test. */ 54 DbInternal.setCheckpointUP(noFileConfig, false); 55 noFileConfig.setConfigParam 56 (EnvironmentParams.LOG_MEMORY_ONLY.getName(), "true"); 57 noFileConfig.setTransactional(true); 58 noFileConfig.setAllowCreate(true); 59 60 env = new Environment(envHome, noFileConfig); 61 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 62 List<String> dbList = envImpl.getDbTree().getDbNames(); 63 assertEquals("no dbs exist", 0, dbList.size()); 64 65 /* Fake a shutdown/startup. */ 66 env.close(); 67 env = new Environment(envHome, noFileConfig); 68 envImpl = DbInternal.getEnvironmentImpl(env); 69 dbList = envImpl.getDbTree().getDbNames(); 70 assertEquals("no dbs exist", 0, dbList.size()); 71 } catch (Throwable t) { 72 t.printStackTrace(); 73 throw t; 74 } finally { 75 if (env != null) 76 env.close(); 77 } 78 } 79 80 /** 81 * Test setting of the database ids in recovery. 82 */ 83 @Test testDbId()84 public void testDbId() 85 throws Throwable { 86 87 Transaction createTxn = null; 88 try { 89 90 /* 91 * Create an environment and three databases. The first two 92 * ids are allocated to the name db and the id db. 93 */ 94 EnvironmentConfig createConfig = TestUtils.initEnvConfig(); 95 createConfig.setTransactional(true); 96 createConfig.setAllowCreate(true); 97 createConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), 98 "6"); 99 env = new Environment(envHome, createConfig); 100 101 int numStartDbs = 1; 102 createTxn = env.beginTransaction(null, null); 103 104 /* Check id of each db. */ 105 DatabaseConfig dbConfig = new DatabaseConfig(); 106 dbConfig.setTransactional(true); 107 dbConfig.setAllowCreate(true); 108 for (int i = 0; i < numStartDbs; i++) { 109 Database anotherDb = env.openDatabase(createTxn, "foo" + i, 110 dbConfig); 111 assertEquals((i+3), 112 DbInternal.getDatabaseImpl(anotherDb). 113 getId().getId()); 114 anotherDb.close(); 115 } 116 createTxn.commit(); 117 env.close(); 118 119 /* 120 * Go through a set of open, creates, and closes. Check id after 121 * recovery. 122 */ 123 EnvironmentConfig envConfig = TestUtils.initEnvConfig(); 124 envConfig.setTransactional(true); 125 createTxn = null; 126 for (int i = numStartDbs; i < numStartDbs + 3; i++) { 127 env = new Environment(envHome, envConfig); 128 129 createTxn = env.beginTransaction(null, null); 130 Database anotherDb = env.openDatabase(createTxn, "foo" + i, 131 dbConfig); 132 assertEquals 133 (i + 3, 134 DbInternal.getDatabaseImpl(anotherDb).getId().getId()); 135 anotherDb.close(); 136 createTxn.commit(); 137 env.close(); 138 } 139 } catch (Throwable t) { 140 if (createTxn != null) { 141 createTxn.abort(); 142 } 143 t.printStackTrace(); 144 throw t; 145 } 146 } 147 148 /** 149 * Test setting the node ids in recovery. 150 */ 151 @Test testNodeId()152 public void testNodeId() 153 throws Throwable { 154 155 try { 156 /* Create an environment and databases. */ 157 createEnvAndDbs(1024, true, NUM_DBS); 158 Map<TestData, Set<TestData>> expectedData = 159 new HashMap<TestData, Set<TestData>>(); 160 161 Transaction txn = env.beginTransaction(null, null); 162 insertData(txn, 0, 4, expectedData, 1, true, NUM_DBS); 163 txn.commit(); 164 165 /* Find the largest node id that has been allocated. */ 166 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 167 DatabaseImpl dbImpl = DbInternal.getDatabaseImpl(dbs[0]); 168 NodeSequence nodeSequence = envImpl.getNodeSequence(); 169 long maxSeenNodeId = nodeSequence.getLastLocalNodeId(); 170 171 /* Close the environment, then recover. */ 172 closeEnv(); 173 EnvironmentConfig recoveryConfig = TestUtils.initEnvConfig(); 174 recoveryConfig.setConfigParam( 175 EnvironmentParams.NODE_MAX.getName(), "6"); 176 recoveryConfig.setConfigParam( 177 EnvironmentParams.ENV_RUN_CLEANER.getName(), 178 "false"); 179 /* Don't checkpoint utilization info for this test. */ 180 DbInternal.setCheckpointUP(recoveryConfig, false); 181 env = new Environment(envHome, recoveryConfig); 182 IN in = new IN(dbImpl, new byte[0], 1, 1); 183 184 /* Recovery should have initialized the next node id to use */ 185 assertTrue("maxSeenNodeId=" + maxSeenNodeId + 186 " in=" + in.getNodeId(), 187 maxSeenNodeId < in.getNodeId()); 188 maxSeenNodeId = nodeSequence.getLastLocalNodeId(); 189 assertEquals(NodeSequence.FIRST_REPLICATED_NODE_ID + 1, 190 nodeSequence.getLastReplicatedNodeId()); 191 192 /* 193 * One more time -- this recovery will get the node id off the 194 * checkpoint of the environment close. This checkpoint records 195 * the fact that the node id was bumped forward by the create of 196 * the IN above. 197 */ 198 env.close(); 199 env = new Environment(envHome, recoveryConfig); 200 in = new IN(dbImpl, new byte[0], 1, 1); 201 /* 202 * The environment re-opening will increment the node id 203 * several times because of the EOF node id. 204 */ 205 assertTrue(maxSeenNodeId < in.getNodeId()); 206 assertEquals(NodeSequence.FIRST_REPLICATED_NODE_ID + 1, 207 nodeSequence.getLastReplicatedNodeId()); 208 209 } catch (Throwable t) { 210 t.printStackTrace(); 211 throw t; 212 } 213 } 214 215 /** 216 * Test setting the txn id. 217 */ 218 @Test 219 public void testTxnId() 220 throws Throwable { 221 222 try { 223 /* Create an environment and databases. */ 224 createEnvAndDbs(1024, true, NUM_DBS); 225 Map<TestData, Set<TestData>> expectedData = 226 new HashMap<TestData, Set<TestData>>(); 227 228 /* Make txns before and after a checkpoint */ 229 Transaction txn = env.beginTransaction(null, null); 230 insertData(txn, 0, 4, expectedData, 1, true, NUM_DBS); 231 txn.commit(); 232 env.checkpoint(forceConfig); 233 txn = env.beginTransaction(null, null); 234 insertData(txn, 5, 6, expectedData, 1, false, NUM_DBS); 235 236 /* Find the largest node id that has been allocated. */ 237 long maxTxnId = txn.getId(); 238 txn.abort(); 239 240 /* Close the environment, then recover. */ 241 closeEnv(); 242 243 EnvironmentConfig recoveryConfig = TestUtils.initEnvConfig(); 244 recoveryConfig.setConfigParam 245 (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); 246 recoveryConfig.setTransactional(true); 247 env = new Environment(envHome, recoveryConfig); 248 249 /* 250 * Check that the next txn id is larger than the last seen. 251 * A few txn ids were eaten by AutoTxns during recovery, do 252 * a basic check that we didn't eat more than 11. 253 */ 254 txn = env.beginTransaction(null, null); 255 createDbs(txn, NUM_DBS); 256 assertTrue(maxTxnId < txn.getId()); 257 assertTrue((txn.getId() - maxTxnId) < 11); 258 259 /* 260 * Do something with this txn so a node with it's value shows up in 261 * the log. 262 */ 263 insertData(txn, 7, 8, expectedData, 1, false, NUM_DBS); 264 long secondMaxTxnId = txn.getId(); 265 txn.abort(); 266 267 /* 268 * One more time -- this recovery will get the txn id off the 269 * checkpoint of the second environment creation. 270 */ 271 closeEnv(); 272 env = new Environment(envHome, recoveryConfig); 273 txn = env.beginTransaction(null, null); 274 assertTrue(secondMaxTxnId < txn.getId()); 275 assertTrue((txn.getId() - secondMaxTxnId) < 10); 276 txn.abort(); 277 } catch (Throwable t) { 278 t.printStackTrace(); 279 throw t; 280 } 281 } 282 283 /** 284 * Test writing a non-transactional db in a transactional environment. 285 * Make sure we can recover. 286 */ 287 @Test 288 public void testNonTxnalDb () 289 throws Throwable { 290 291 createEnv(1024, false); 292 try { 293 294 /* 295 * Create a database, write into it non-txnally. Should be 296 * allowed 297 */ 298 DatabaseConfig dbConfig = new DatabaseConfig(); 299 dbConfig.setAllowCreate(true); 300 Database dbA = env.openDatabase(null, "NotTxnal", dbConfig); 301 302 DatabaseEntry key = new StringDbt("foo"); 303 DatabaseEntry data = new StringDbt("bar"); 304 dbA.put(null, key, data); 305 306 /* close and recover -- the database should still be there 307 * because we're shutting down clean. 308 */ 309 dbA.close(); 310 env.close(); 311 createEnv(1024, false); 312 313 dbA = env.openDatabase(null, "NotTxnal", null); 314 dbA.close(); 315 316 /* 317 * Create a database, auto commit. Then write a record. 318 * The database should exist after recovery. 319 */ 320 dbConfig.setTransactional(true); 321 Database dbB = env.openDatabase(null, "Txnal", dbConfig); 322 dbB.close(); 323 dbB = env.openDatabase(null, "Txnal", null); 324 dbB.put(null, key, data); 325 dbB.close(); 326 env.close(); 327 328 /* 329 * Recover. We should see the database. We may or may not see 330 * the records. 331 */ 332 createEnv(1024, false); 333 List<String> dbNames = env.getDatabaseNames(); 334 assertEquals(2, dbNames.size()); 335 assertEquals("Txnal", dbNames.get(1)); 336 assertEquals("NotTxnal", dbNames.get(0)); 337 338 } catch (Throwable t) { 339 t.printStackTrace(); 340 throw t; 341 } finally { 342 env.close(); 343 } 344 } 345 346 /** 347 * Test that we can recover with a bad checksum. 348 */ 349 @Test 350 public void testBadChecksum() 351 throws Throwable { 352 353 try { 354 /* Create an environment and databases. */ 355 createEnvAndDbs(2048, false, 1); 356 Map<TestData, Set<TestData>> expectedData = 357 new HashMap<TestData, Set<TestData>>(); 358 359 /* Make txns before and after a checkpoint */ 360 Transaction txn = env.beginTransaction(null, null); 361 insertData(txn, 0, 4, expectedData, 1, true, 1); 362 txn.commit(); 363 env.checkpoint(forceConfig); 364 365 txn = env.beginTransaction(null, null); 366 insertData(txn, 5, 6, expectedData, 1, true, 1); 367 txn.commit(); 368 369 txn = env.beginTransaction(null, null); 370 insertData(txn, 7, 8, expectedData, 1, false, 1); 371 372 /* Close the environment, then recover. */ 373 closeEnv(); 374 375 /* Write some 0's into the last file. */ 376 writeBadStuffInLastFile(); 377 378 recoverAndVerify(expectedData, 1); 379 } catch (Throwable t) { 380 t.printStackTrace(); 381 throw t; 382 } 383 } 384 385 /** 386 * Another bad checksum test. Make sure that there is no checkpoint in the 387 * last file so that this recovery will have to read backwards into the 388 * previous file. Also recover in read/only mode to make sure we don't 389 * process the bad portion of the log. 390 */ 391 @Test 392 public void testBadChecksumReadOnlyReadPastLastFile() 393 throws Throwable { 394 395 try { 396 /* Create an environment and databases. */ 397 createEnvAndDbs(500, false, 1); 398 Map<TestData, Set<TestData>> expectedData = 399 new HashMap<TestData, Set<TestData>>(); 400 401 /* Commit some data, checkpoint. */ 402 Transaction txn = env.beginTransaction(null, null); 403 insertData(txn, 0, 4, expectedData, 1, true, 1); 404 txn.commit(); 405 env.checkpoint(forceConfig); 406 407 /* 408 * Remember how many files we have, so we know where the last 409 * checkpoint is. 410 */ 411 String[] suffixes = new String[] {FileManager.JE_SUFFIX}; 412 String[] fileList = 413 FileManager.listFiles(envHome, suffixes, false); 414 int startingNumFiles = fileList.length; 415 416 /* Now add enough non-committed data to add more files. */ 417 txn = env.beginTransaction(null, null); 418 insertData(txn, 7, 50, expectedData, 1, false, 1); 419 420 /* Close the environment, then recover. */ 421 closeEnv(); 422 423 /* Make sure that we added on files after the checkpoint. */ 424 fileList = FileManager.listFiles(envHome, suffixes, false); 425 assertTrue(fileList.length > startingNumFiles); 426 427 /* Write some 0's into the last file. */ 428 writeBadStuffInLastFile(); 429 430 recoverROAndVerify(expectedData, 1); 431 } catch (Throwable t) { 432 t.printStackTrace(); 433 throw t; 434 } 435 } 436 437 private void writeBadStuffInLastFile() 438 throws IOException { 439 440 String[] files = 441 FileManager.listFiles(envHome, 442 new String[] {FileManager.JE_SUFFIX}, 443 false); 444 File lastFile = new File(envHome, files[files.length - 1]); 445 RandomAccessFile rw = new RandomAccessFile(lastFile, "rw"); 446 447 rw.seek(rw.length() - 10); 448 rw.writeBytes("000000"); 449 rw.close(); 450 } 451 452 /** 453 * Test that we can recover with no checkpoint end 454 */ 455 @Test 456 public void testNoCheckpointEnd() 457 throws Exception { 458 459 /* Create a new environment */ 460 EnvironmentConfig createConfig = TestUtils.initEnvConfig(); 461 createConfig.setTransactional(true); 462 createConfig.setAllowCreate(true); 463 env = new Environment(envHome, createConfig); 464 465 /* Truncate before the first ckpt end. */ 466 truncateAtEntry(LogEntryType.LOG_CKPT_END); 467 env.close(); 468 469 /* Check that we can recover. */ 470 createConfig.setAllowCreate(false); 471 env = new Environment(envHome, createConfig); 472 env.close(); 473 } 474 475 /** 476 * Truncate the log so it doesn't include the first incidence of this 477 * log entry type. 478 */ 479 private void truncateAtEntry(LogEntryType entryType) 480 throws Exception { 481 482 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 483 484 /* 485 * Find the first given log entry type and truncate the file so it 486 * doesn't include that entry. 487 */ 488 SearchFileReader reader = 489 new SearchFileReader(envImpl, 490 1000, // readBufferSize 491 true, // forward 492 0, // startLSN 493 DbLsn.NULL_LSN, // endLSN 494 entryType); 495 496 long targetLsn = 0; 497 if (reader.readNextEntry()) { 498 targetLsn = reader.getLastLsn(); 499 } else { 500 fail("There should be some kind of " + entryType + " in the log."); 501 } 502 503 assertTrue(targetLsn != 0); 504 envImpl.getFileManager().truncateLog(DbLsn.getFileNumber(targetLsn), 505 DbLsn.getFileOffset(targetLsn)); 506 } 507 } 508