1 /*- 2 * See the file LICENSE for redistribution information. 3 * 4 * Copyright (c) 2002, 2014 Oracle and/or its affiliates. All rights reserved. 5 * 6 */ 7 8 package com.sleepycat.je.cleaner; 9 10 11 import static org.junit.Assert.assertEquals; 12 import static org.junit.Assert.assertTrue; 13 14 import java.io.IOException; 15 16 import org.junit.After; 17 import org.junit.Test; 18 19 import com.sleepycat.bind.tuple.IntegerBinding; 20 import com.sleepycat.je.CheckpointConfig; 21 import com.sleepycat.je.Cursor; 22 import com.sleepycat.je.Database; 23 import com.sleepycat.je.DatabaseConfig; 24 import com.sleepycat.je.DatabaseEntry; 25 import com.sleepycat.je.DatabaseException; 26 import com.sleepycat.je.DbInternal; 27 import com.sleepycat.je.Environment; 28 import com.sleepycat.je.EnvironmentConfig; 29 import com.sleepycat.je.Transaction; 30 import com.sleepycat.je.config.EnvironmentParams; 31 import com.sleepycat.je.dbi.DatabaseImpl; 32 import com.sleepycat.je.dbi.EnvironmentImpl; 33 import com.sleepycat.je.log.LogEntryType; 34 import com.sleepycat.je.log.SearchFileReader; 35 import com.sleepycat.je.tree.IN; 36 import com.sleepycat.je.tree.MapLN; 37 import com.sleepycat.je.util.TestUtils; 38 import com.sleepycat.je.utilint.DbLsn; 39 40 /** 41 * Test utilization counting of INs. 42 */ 43 public class INUtilizationTest extends CleanerTestBase { 44 45 private static final String DB_NAME = "foo"; 46 47 private static final CheckpointConfig forceConfig = new CheckpointConfig(); 48 static { 49 forceConfig.setForce(true); 50 } 51 52 private EnvironmentImpl envImpl; 53 private Database db; 54 private DatabaseImpl dbImpl; 55 private Transaction txn; 56 private Cursor cursor; 57 private boolean dups = false; 58 private DatabaseEntry keyEntry = new DatabaseEntry(); 59 private DatabaseEntry dataEntry = new DatabaseEntry(); 60 private boolean truncateOrRemoveDone; 61 INUtilizationTest()62 public INUtilizationTest() { 63 envMultiSubDir = false; 64 } 65 66 @After tearDown()67 public void tearDown() 68 throws Exception { 69 70 super.tearDown(); 71 envImpl = null; 72 db = null; 73 dbImpl = null; 74 txn = null; 75 cursor = null; 76 keyEntry = null; 77 dataEntry = null; 78 } 79 80 /** 81 * Opens the environment and database. 82 */ 83 @SuppressWarnings("deprecation") openEnv()84 private void openEnv() 85 throws DatabaseException { 86 87 EnvironmentConfig config = TestUtils.initEnvConfig(); 88 DbInternal.disableParameterValidation(config); 89 config.setTransactional(true); 90 config.setTxnNoSync(true); 91 config.setAllowCreate(true); 92 /* Do not run the daemons. */ 93 config.setConfigParam 94 (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); 95 config.setConfigParam 96 (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false"); 97 config.setConfigParam 98 (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); 99 config.setConfigParam 100 (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false"); 101 /* Use a tiny log file size to write one node per file. */ 102 config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(), 103 Integer.toString(64)); 104 if (envMultiSubDir) { 105 config.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, 106 DATA_DIRS + ""); 107 } 108 env = new Environment(envHome, config); 109 envImpl = DbInternal.getEnvironmentImpl(env); 110 111 /* Speed up test that uses lots of very small files. */ 112 envImpl.getFileManager().setSyncAtFileEnd(false); 113 114 openDb(); 115 } 116 117 /** 118 * Opens the database. 119 */ openDb()120 private void openDb() 121 throws DatabaseException { 122 123 DatabaseConfig dbConfig = new DatabaseConfig(); 124 dbConfig.setTransactional(true); 125 dbConfig.setAllowCreate(true); 126 dbConfig.setSortedDuplicates(dups); 127 db = env.openDatabase(null, DB_NAME, dbConfig); 128 dbImpl = DbInternal.getDatabaseImpl(db); 129 } 130 closeEnv(boolean doCheckpoint)131 private void closeEnv(boolean doCheckpoint) 132 throws DatabaseException { 133 134 closeEnv(doCheckpoint, 135 true, // expectAccurateObsoleteLNCount 136 true); // expectAccurateObsoleteLNSize 137 } 138 closeEnv(boolean doCheckpoint, boolean expectAccurateObsoleteLNCount)139 private void closeEnv(boolean doCheckpoint, 140 boolean expectAccurateObsoleteLNCount) 141 throws DatabaseException { 142 143 closeEnv(doCheckpoint, 144 expectAccurateObsoleteLNCount, 145 expectAccurateObsoleteLNCount); 146 } 147 148 /** 149 * Closes the environment and database. 150 * 151 * @param expectAccurateObsoleteLNCount should be false when a deleted LN 152 * is not counted properly by recovery because its parent INs were flushed 153 * and the obsolete LN was not found in the tree. 154 * 155 * @param expectAccurateObsoleteLNSize should be false when a tree walk is 156 * performed for truncate/remove or an abortLsn is counted by recovery. 157 */ closeEnv(boolean doCheckpoint, boolean expectAccurateObsoleteLNCount, boolean expectAccurateObsoleteLNSize)158 private void closeEnv(boolean doCheckpoint, 159 boolean expectAccurateObsoleteLNCount, 160 boolean expectAccurateObsoleteLNSize) 161 throws DatabaseException { 162 163 /* 164 * We pass expectAccurateDbUtilization as false when 165 * truncateOrRemoveDone, because the database utilization info for that 166 * database is now gone. 167 */ 168 VerifyUtils.verifyUtilization 169 (envImpl, expectAccurateObsoleteLNCount, 170 expectAccurateObsoleteLNSize, 171 !truncateOrRemoveDone); // expectAccurateDbUtilization 172 173 if (db != null) { 174 db.close(); 175 db = null; 176 dbImpl = null; 177 } 178 if (envImpl != null) { 179 envImpl.close(doCheckpoint); 180 envImpl = null; 181 env = null; 182 } 183 } 184 185 /** 186 * Initial setup for all tests -- open env, put one record (or two for 187 * dups) and sync. 188 */ openAndWriteDatabase()189 private void openAndWriteDatabase() 190 throws DatabaseException { 191 192 openEnv(); 193 txn = env.beginTransaction(null, null); 194 cursor = db.openCursor(txn, null); 195 196 /* Put one record. */ 197 IntegerBinding.intToEntry(0, keyEntry); 198 IntegerBinding.intToEntry(0, dataEntry); 199 cursor.put(keyEntry, dataEntry); 200 201 /* Add a duplicate. */ 202 if (dups) { 203 IntegerBinding.intToEntry(1, dataEntry); 204 cursor.put(keyEntry, dataEntry); 205 } 206 207 /* Commit the txn to avoid crossing the checkpoint boundary. */ 208 cursor.close(); 209 txn.commit(); 210 211 /* Checkpoint to the root so nothing is dirty. */ 212 env.sync(); 213 214 /* Open a txn and cursor for use by the test case. */ 215 txn = env.beginTransaction(null, null); 216 cursor = db.openCursor(txn, null); 217 218 /* If we added a duplicate, move cursor back to the first record. */ 219 cursor.getFirst(keyEntry, dataEntry, null); 220 221 /* Expect that BIN and parent IN files are not obsolete. */ 222 long binFile = getBINFile(cursor); 223 long inFile = getINFile(cursor); 224 expectObsolete(binFile, false); 225 expectObsolete(inFile, false); 226 } 227 228 /** 229 * Tests that BIN and IN utilization counting works. 230 */ 231 @Test testBasic()232 public void testBasic() 233 throws DatabaseException { 234 235 openAndWriteDatabase(); 236 237 long binFile = getBINFile(cursor); 238 long inFile = getINFile(cursor); 239 240 /* Update to make BIN dirty. */ 241 cursor.put(keyEntry, dataEntry); 242 243 /* Checkpoint */ 244 env.checkpoint(forceConfig); 245 246 /* After checkpoint, expect BIN and IN are obsolete. */ 247 expectObsolete(binFile, true); 248 expectObsolete(inFile, true); 249 assertTrue(binFile != getBINFile(cursor)); 250 assertTrue(inFile != getINFile(cursor)); 251 252 /* After second checkpoint, no changes. */ 253 env.checkpoint(forceConfig); 254 255 /* Both BIN and IN are obsolete. */ 256 expectObsolete(binFile, true); 257 expectObsolete(inFile, true); 258 assertTrue(binFile != getBINFile(cursor)); 259 assertTrue(inFile != getINFile(cursor)); 260 261 /* Expect that new files are not obsolete. */ 262 long binFile2 = getBINFile(cursor); 263 long inFile2 = getINFile(cursor); 264 expectObsolete(binFile2, false); 265 expectObsolete(inFile2, false); 266 267 cursor.close(); 268 txn.commit(); 269 closeEnv(true); 270 } 271 272 /** 273 * Performs testBasic with duplicates. 274 */ 275 @Test testBasicDup()276 public void testBasicDup() 277 throws DatabaseException { 278 279 dups = true; 280 testBasic(); 281 } 282 283 /** 284 * Tests that BIN-delta utilization counting works. 285 */ 286 @Test testBINDeltas()287 public void testBINDeltas() 288 throws DatabaseException { 289 290 /* 291 * Insert 4 additional records so there will be 5 slots total. Then one 292 * modified slot (less than 25% of the total) will cause a delta to be 293 * logged. Two modified slots (more than 25% of the total) will cause 294 * a full version to be logged. 295 */ 296 openAndWriteDatabase(); 297 for (int i = 1; i <= 4; i += 1) { 298 IntegerBinding.intToEntry(i, keyEntry); 299 IntegerBinding.intToEntry(0, dataEntry); 300 db.put(txn, keyEntry, dataEntry); 301 } 302 env.sync(); 303 long fullBinFile = getFullBINFile(cursor); 304 long deltaBinFile = getDeltaBINFile(cursor); 305 long inFile = getINFile(cursor); 306 assertEquals(-1, deltaBinFile); 307 308 /* Update first record to make one BIN slot dirty. */ 309 IntegerBinding.intToEntry(0, keyEntry); 310 cursor.put(keyEntry, dataEntry); 311 312 /* Checkpoint, write BIN-delta. */ 313 env.checkpoint(forceConfig); 314 315 /* After checkpoint, expect only IN is obsolete. */ 316 expectObsolete(fullBinFile, false); 317 expectObsolete(inFile, true); 318 assertTrue(fullBinFile == getFullBINFile(cursor)); 319 assertTrue(deltaBinFile != getDeltaBINFile(cursor)); 320 assertTrue(inFile != getINFile(cursor)); 321 322 /* After second checkpoint, no changes. */ 323 env.checkpoint(forceConfig); 324 expectObsolete(fullBinFile, false); 325 expectObsolete(inFile, true); 326 assertTrue(fullBinFile == getFullBINFile(cursor)); 327 assertTrue(deltaBinFile != getDeltaBINFile(cursor)); 328 assertTrue(inFile != getINFile(cursor)); 329 330 fullBinFile = getFullBINFile(cursor); 331 deltaBinFile = getDeltaBINFile(cursor); 332 inFile = getINFile(cursor); 333 assertTrue(deltaBinFile != -1); 334 335 /* Update first record again, checkpoint to write another delta. */ 336 IntegerBinding.intToEntry(0, keyEntry); 337 cursor.put(keyEntry, dataEntry); 338 339 /* After checkpoint, expect IN and first delta are obsolete. */ 340 env.checkpoint(forceConfig); 341 expectObsolete(fullBinFile, false); 342 expectObsolete(deltaBinFile, true); 343 expectObsolete(inFile, true); 344 assertTrue(fullBinFile == getFullBINFile(cursor)); 345 assertTrue(deltaBinFile != getDeltaBINFile(cursor)); 346 assertTrue(inFile != getINFile(cursor)); 347 348 fullBinFile = getFullBINFile(cursor); 349 deltaBinFile = getDeltaBINFile(cursor); 350 inFile = getINFile(cursor); 351 assertTrue(deltaBinFile != -1); 352 353 /* Update two records, checkpoint to write a full BIN version. */ 354 IntegerBinding.intToEntry(0, keyEntry); 355 cursor.put(keyEntry, dataEntry); 356 IntegerBinding.intToEntry(1, keyEntry); 357 cursor.put(keyEntry, dataEntry); 358 359 /* After checkpoint, expect IN, full BIN, last delta are obsolete. */ 360 env.checkpoint(forceConfig); 361 expectObsolete(fullBinFile, true); 362 expectObsolete(deltaBinFile, true); 363 expectObsolete(inFile, true); 364 assertTrue(fullBinFile != getFullBINFile(cursor)); 365 assertTrue(deltaBinFile != getDeltaBINFile(cursor)); 366 assertTrue(inFile != getINFile(cursor)); 367 assertEquals(-1, getDeltaBINFile(cursor)); 368 369 /* Expect that new files are not obsolete. */ 370 long binFile2 = getBINFile(cursor); 371 long inFile2 = getINFile(cursor); 372 expectObsolete(binFile2, false); 373 expectObsolete(inFile2, false); 374 375 cursor.close(); 376 txn.commit(); 377 closeEnv(true); 378 } 379 380 /** 381 * Performs testBINDeltas with duplicates. 382 */ 383 @Test testBINDeltasDup()384 public void testBINDeltasDup() 385 throws DatabaseException { 386 387 dups = true; 388 testBINDeltas(); 389 } 390 391 /** 392 * Similar to testBasic, but logs INs explicitly and performs recovery to 393 * ensure utilization recovery works. 394 */ 395 @Test testRecovery()396 public void testRecovery() 397 throws DatabaseException { 398 399 openAndWriteDatabase(); 400 long binFile = getBINFile(cursor); 401 long inFile = getINFile(cursor); 402 403 /* Close normally and reopen. */ 404 cursor.close(); 405 txn.commit(); 406 closeEnv(true); 407 openEnv(); 408 txn = env.beginTransaction(null, null); 409 cursor = db.openCursor(txn, null); 410 411 /* Position cursor to load BIN and IN. */ 412 cursor.getSearchKey(keyEntry, dataEntry, null); 413 414 /* Expect BIN and IN files have not changed. */ 415 assertEquals(binFile, getBINFile(cursor)); 416 assertEquals(inFile, getINFile(cursor)); 417 expectObsolete(binFile, false); 418 expectObsolete(inFile, false); 419 420 /* 421 * Log explicitly since we have no way to do a partial checkpoint. 422 * The BIN is logged provisionally and the IN non-provisionally. 423 */ 424 TestUtils.logBINAndIN(env, cursor); 425 426 /* Expect to obsolete the BIN and IN. */ 427 expectObsolete(binFile, true); 428 expectObsolete(inFile, true); 429 assertTrue(binFile != getBINFile(cursor)); 430 assertTrue(inFile != getINFile(cursor)); 431 432 /* Save current BIN and IN files. */ 433 long binFile2 = getBINFile(cursor); 434 long inFile2 = getINFile(cursor); 435 expectObsolete(binFile2, false); 436 expectObsolete(inFile2, false); 437 438 /* Shutdown without a checkpoint and reopen. */ 439 cursor.close(); 440 txn.commit(); 441 closeEnv(false); 442 openEnv(); 443 txn = env.beginTransaction(null, null); 444 cursor = db.openCursor(txn, null); 445 446 /* Sync to make all INs non-dirty. */ 447 env.sync(); 448 449 /* Position cursor to load BIN and IN. */ 450 cursor.getSearchKey(keyEntry, dataEntry, null); 451 452 /* Expect that recovery counts BIN and IN as obsolete. */ 453 expectObsolete(binFile, true); 454 expectObsolete(inFile, true); 455 assertTrue(binFile != getBINFile(cursor)); 456 assertTrue(inFile != getINFile(cursor)); 457 458 /* 459 * Even though it is provisional, expect that current BIN is not 460 * obsolete because it is not part of partial checkpoint. This is 461 * similar to what happens with a split. The current IN is not 462 * obsolete either (nor is it provisional). 463 */ 464 assertTrue(binFile2 == getBINFile(cursor)); 465 assertTrue(inFile2 == getINFile(cursor)); 466 expectObsolete(binFile2, false); 467 expectObsolete(inFile2, false); 468 469 /* Update to make BIN dirty. */ 470 cursor.put(keyEntry, dataEntry); 471 472 /* Check current BIN and IN files. */ 473 assertTrue(binFile2 == getBINFile(cursor)); 474 assertTrue(inFile2 == getINFile(cursor)); 475 expectObsolete(binFile2, false); 476 expectObsolete(inFile2, false); 477 478 /* Close normally and reopen to cause checkpoint of dirty BIN/IN. */ 479 cursor.close(); 480 txn.commit(); 481 closeEnv(true); 482 openEnv(); 483 txn = env.beginTransaction(null, null); 484 cursor = db.openCursor(txn, null); 485 486 /* Position cursor to load BIN and IN. */ 487 cursor.getSearchKey(keyEntry, dataEntry, null); 488 489 /* Expect BIN and IN were checkpointed during close. */ 490 assertTrue(binFile2 != getBINFile(cursor)); 491 assertTrue(inFile2 != getINFile(cursor)); 492 expectObsolete(binFile2, true); 493 expectObsolete(inFile2, true); 494 495 /* After second checkpoint, no change. */ 496 env.checkpoint(forceConfig); 497 498 /* Both BIN and IN are obsolete. */ 499 assertTrue(binFile2 != getBINFile(cursor)); 500 assertTrue(inFile2 != getINFile(cursor)); 501 expectObsolete(binFile2, true); 502 expectObsolete(inFile2, true); 503 504 cursor.close(); 505 txn.commit(); 506 closeEnv(true); 507 } 508 509 /** 510 * Performs testRecovery with duplicates. 511 */ 512 @Test testRecoveryDup()513 public void testRecoveryDup() 514 throws DatabaseException { 515 516 dups = true; 517 testRecovery(); 518 } 519 520 /** 521 * Similar to testRecovery, but tests BIN-deltas. 522 */ 523 @Test testBINDeltaRecovery()524 public void testBINDeltaRecovery() 525 throws DatabaseException { 526 527 /* 528 * Insert 4 additional records so there will be 5 slots total. Then one 529 * modified slot (less than 25% of the total) will cause a delta to be 530 * logged. Two modified slots (more than 25% of the total) will cause 531 * a full version to be logged. 532 */ 533 openAndWriteDatabase(); 534 for (int i = 1; i <= 4; i += 1) { 535 IntegerBinding.intToEntry(i, keyEntry); 536 IntegerBinding.intToEntry(0, dataEntry); 537 db.put(txn, keyEntry, dataEntry); 538 } 539 env.sync(); 540 long fullBinFile = getFullBINFile(cursor); 541 long deltaBinFile = getDeltaBINFile(cursor); 542 long inFile = getINFile(cursor); 543 assertEquals(-1, deltaBinFile); 544 545 /* Close normally and reopen. */ 546 cursor.close(); 547 txn.commit(); 548 closeEnv(true); 549 openEnv(); 550 txn = env.beginTransaction(null, null); 551 cursor = db.openCursor(txn, null); 552 553 /* Position cursor to load BIN and IN. */ 554 cursor.getSearchKey(keyEntry, dataEntry, null); 555 556 /* Expect BIN and IN files have not changed. */ 557 assertEquals(fullBinFile, getFullBINFile(cursor)); 558 assertEquals(deltaBinFile, getDeltaBINFile(cursor)); 559 assertEquals(inFile, getINFile(cursor)); 560 expectObsolete(fullBinFile, false); 561 expectObsolete(inFile, false); 562 563 /* Update first record to make one BIN slot dirty. */ 564 IntegerBinding.intToEntry(0, keyEntry); 565 cursor.put(keyEntry, dataEntry); 566 567 /* 568 * Log explicitly since we have no way to do a partial checkpoint. 569 * The BIN-delta is logged provisionally and the IN non-provisionally. 570 */ 571 TestUtils.logBINAndIN(env, cursor, true /*allowDeltas*/); 572 573 /* Expect to obsolete the IN but not the full BIN. */ 574 expectObsolete(fullBinFile, false); 575 expectObsolete(inFile, true); 576 assertEquals(fullBinFile, getFullBINFile(cursor)); 577 578 /* Save current BIN-delta and IN files. */ 579 long deltaBinFile2 = getDeltaBINFile(cursor); 580 long inFile2 = getINFile(cursor); 581 assertTrue(deltaBinFile != deltaBinFile2); 582 assertTrue(inFile != inFile2); 583 expectObsolete(deltaBinFile2, false); 584 expectObsolete(inFile2, false); 585 586 /* Shutdown without a checkpoint and reopen. */ 587 cursor.close(); 588 txn.commit(); 589 closeEnv(false, // doCheckpoint 590 true, // expectAccurateObsoleteLNCount 591 false); // expectAccurateObsoleteLNSize 592 openEnv(); 593 txn = env.beginTransaction(null, null); 594 cursor = db.openCursor(txn, null); 595 596 /* Sync to make all INs non-dirty. */ 597 env.sync(); 598 599 /* Position cursor to load BIN and IN. */ 600 cursor.getSearchKey(keyEntry, dataEntry, null); 601 602 /* Expect that recovery counts only IN as obsolete. */ 603 expectObsolete(inFile, true); 604 expectObsolete(fullBinFile, false); 605 expectObsolete(deltaBinFile2, false); 606 expectObsolete(inFile2, false); 607 assertEquals(fullBinFile, getFullBINFile(cursor)); 608 assertEquals(deltaBinFile2, getDeltaBINFile(cursor)); 609 assertEquals(inFile2, getINFile(cursor)); 610 611 /* Reset variables to current versions. */ 612 deltaBinFile = deltaBinFile2; 613 inFile = inFile2; 614 615 /* Update same slot and write another delta. */ 616 IntegerBinding.intToEntry(0, keyEntry); 617 cursor.put(keyEntry, dataEntry); 618 TestUtils.logBINAndIN(env, cursor, true /*allowDeltas*/); 619 620 /* Expect to obsolete the BIN-delta and IN, but not the full BIN. */ 621 expectObsolete(fullBinFile, false); 622 expectObsolete(deltaBinFile, true); 623 expectObsolete(inFile, true); 624 assertEquals(fullBinFile, getFullBINFile(cursor)); 625 626 /* Save current BIN-delta and IN files. */ 627 deltaBinFile2 = getDeltaBINFile(cursor); 628 inFile2 = getINFile(cursor); 629 assertTrue(deltaBinFile != deltaBinFile2); 630 assertTrue(inFile != inFile2); 631 expectObsolete(deltaBinFile2, false); 632 expectObsolete(inFile2, false); 633 634 /* Shutdown without a checkpoint and reopen. */ 635 cursor.close(); 636 txn.commit(); 637 closeEnv(false, // doCheckpoint 638 true, // expectAccurateObsoleteLNCount 639 false); // expectAccurateObsoleteLNSize 640 openEnv(); 641 txn = env.beginTransaction(null, null); 642 cursor = db.openCursor(txn, null); 643 644 /* Sync to make all INs non-dirty. */ 645 env.sync(); 646 647 /* Position cursor to load BIN and IN. */ 648 cursor.getSearchKey(keyEntry, dataEntry, null); 649 650 /* Expect that recovery counts only BIN-delta and IN as obsolete. */ 651 expectObsolete(fullBinFile, false); 652 expectObsolete(deltaBinFile, true); 653 expectObsolete(inFile, true); 654 expectObsolete(deltaBinFile2, false); 655 expectObsolete(inFile2, false); 656 assertEquals(fullBinFile, getFullBINFile(cursor)); 657 assertEquals(deltaBinFile2, getDeltaBINFile(cursor)); 658 assertEquals(inFile2, getINFile(cursor)); 659 660 /* Reset variables to current versions. */ 661 deltaBinFile = deltaBinFile2; 662 inFile = inFile2; 663 664 /* Update two records and write a full BIN version. */ 665 IntegerBinding.intToEntry(0, keyEntry); 666 cursor.put(keyEntry, dataEntry); 667 IntegerBinding.intToEntry(1, keyEntry); 668 cursor.put(keyEntry, dataEntry); 669 TestUtils.logBINAndIN(env, cursor, true /*allowDeltas*/); 670 671 /* Expect to obsolete the full BIN, the BIN-delta and the IN. */ 672 expectObsolete(fullBinFile, true); 673 expectObsolete(deltaBinFile, true); 674 expectObsolete(inFile, true); 675 676 /* Save current BIN, BIN-delta and IN files. */ 677 long fullBinFile2 = getFullBINFile(cursor); 678 deltaBinFile2 = getDeltaBINFile(cursor); 679 inFile2 = getINFile(cursor); 680 assertTrue(fullBinFile != fullBinFile2); 681 assertTrue(deltaBinFile != deltaBinFile2); 682 assertTrue(inFile != inFile2); 683 assertEquals(DbLsn.NULL_LSN, deltaBinFile2); 684 expectObsolete(fullBinFile2, false); 685 expectObsolete(inFile2, false); 686 687 /* Shutdown without a checkpoint and reopen. */ 688 cursor.close(); 689 txn.commit(); 690 closeEnv(false, // doCheckpoint 691 true, // expectAccurateObsoleteLNCount 692 false); // expectAccurateObsoleteLNSize 693 openEnv(); 694 txn = env.beginTransaction(null, null); 695 cursor = db.openCursor(txn, null); 696 697 /* Sync to make all INs non-dirty. */ 698 env.sync(); 699 700 /* Position cursor to load BIN and IN. */ 701 cursor.getSearchKey(keyEntry, dataEntry, null); 702 703 /* Expect that recovery counts BIN, BIN-delta and IN as obsolete. */ 704 expectObsolete(fullBinFile, true); 705 expectObsolete(deltaBinFile, true); 706 expectObsolete(inFile, true); 707 expectObsolete(fullBinFile2, false); 708 assertEquals(DbLsn.NULL_LSN, deltaBinFile2); 709 expectObsolete(inFile2, false); 710 assertEquals(deltaBinFile2, getDeltaBINFile(cursor)); 711 assertEquals(inFile2, getINFile(cursor)); 712 713 cursor.close(); 714 txn.commit(); 715 closeEnv(false, // doCheckpoint 716 true, // expectAccurateObsoleteLNCount 717 false); // expectAccurateObsoleteLNSize 718 } 719 720 /** 721 * Performs testRecovery with duplicates. 722 */ 723 @Test testBINDeltaRecoveryDup()724 public void testBINDeltaRecoveryDup() 725 throws DatabaseException { 726 727 dups = true; 728 testBINDeltaRecovery(); 729 } 730 731 /** 732 * Tests that in a partial checkpoint (CkptStart with no CkptEnd) all 733 * provisional INs are counted as obsolete. 734 */ 735 @Test testPartialCheckpoint()736 public void testPartialCheckpoint() 737 throws DatabaseException, IOException { 738 739 openAndWriteDatabase(); 740 long binFile = getBINFile(cursor); 741 long inFile = getINFile(cursor); 742 743 /* Close with partial checkpoint and reopen. */ 744 cursor.close(); 745 txn.commit(); 746 performPartialCheckpoint(true); // truncateUtilizationInfo 747 748 openEnv(); 749 txn = env.beginTransaction(null, null); 750 cursor = db.openCursor(txn, null); 751 752 /* Position cursor to load BIN and IN. */ 753 cursor.getSearchKey(keyEntry, dataEntry, null); 754 755 /* Expect BIN and IN files have not changed. */ 756 assertEquals(binFile, getBINFile(cursor)); 757 assertEquals(inFile, getINFile(cursor)); 758 expectObsolete(binFile, false); 759 expectObsolete(inFile, false); 760 761 /* Update to make BIN dirty. */ 762 cursor.put(keyEntry, dataEntry); 763 764 /* Force IN dirty so that BIN is logged provisionally. */ 765 TestUtils.getIN(TestUtils.getBIN(cursor)).setDirty(true); 766 767 /* Check current BIN and IN files. */ 768 assertTrue(binFile == getBINFile(cursor)); 769 assertTrue(inFile == getINFile(cursor)); 770 expectObsolete(binFile, false); 771 expectObsolete(inFile, false); 772 773 /* Close with partial checkpoint and reopen. */ 774 cursor.close(); 775 txn.commit(); 776 performPartialCheckpoint(true); // truncateUtilizationInfo 777 openEnv(); 778 txn = env.beginTransaction(null, null); 779 cursor = db.openCursor(txn, null); 780 781 /* Position cursor to load BIN and IN. */ 782 cursor.getSearchKey(keyEntry, dataEntry, null); 783 784 /* Expect BIN and IN files are obsolete. */ 785 assertTrue(binFile != getBINFile(cursor)); 786 assertTrue(inFile != getINFile(cursor)); 787 expectObsolete(binFile, true); 788 expectObsolete(inFile, true); 789 790 /* 791 * Expect that the current BIN is obsolete because it was provisional, 792 * and provisional nodes following CkptStart are counted obsolete 793 * even if that is sometimes incorrect. The parent IN file is not 794 * obsolete because it is not provisonal. 795 */ 796 long binFile2 = getBINFile(cursor); 797 long inFile2 = getINFile(cursor); 798 expectObsolete(binFile2, true); 799 expectObsolete(inFile2, false); 800 801 /* 802 * Now repeat the test above but do not truncate the FileSummaryLNs. 803 * The counting will be accurate because the FileSummaryLNs override 804 * what is counted manually during recovery. 805 */ 806 807 /* Update to make BIN dirty. */ 808 cursor.put(keyEntry, dataEntry); 809 810 /* Close with partial checkpoint and reopen. */ 811 cursor.close(); 812 txn.commit(); 813 performPartialCheckpoint(false, // truncateUtilizationInfo 814 true, // expectAccurateObsoleteLNCount 815 false); // expectAccurateObsoleteLNSize 816 817 openEnv(); 818 txn = env.beginTransaction(null, null); 819 cursor = db.openCursor(txn, null); 820 821 /* Position cursor to load BIN and IN. */ 822 cursor.getSearchKey(keyEntry, dataEntry, null); 823 824 /* The prior BIN file is now double-counted as obsolete. */ 825 assertTrue(binFile2 != getBINFile(cursor)); 826 assertTrue(inFile2 != getINFile(cursor)); 827 expectObsolete(binFile2, 2); 828 expectObsolete(inFile2, 1); 829 830 /* Expect current BIN and IN files are not obsolete. */ 831 binFile2 = getBINFile(cursor); 832 inFile2 = getINFile(cursor); 833 expectObsolete(binFile2, false); 834 expectObsolete(inFile2, false); 835 836 cursor.close(); 837 txn.commit(); 838 closeEnv(true, // doCheckpoint 839 true, // expectAccurateObsoleteLNCount 840 false); // expectAccurateObsoleteLNSize 841 } 842 843 /** 844 * Performs testPartialCheckpoint with duplicates. 845 */ 846 @Test testPartialCheckpointDup()847 public void testPartialCheckpointDup() 848 throws DatabaseException, IOException { 849 850 dups = true; 851 testPartialCheckpoint(); 852 } 853 854 /** 855 * Tests that deleting a subtree (by deleting the last LN in a BIN) is 856 * counted correctly. 857 */ 858 @Test testDelete()859 public void testDelete() 860 throws DatabaseException, IOException { 861 862 openAndWriteDatabase(); 863 long binFile = getBINFile(cursor); 864 long inFile = getINFile(cursor); 865 866 /* Close normally and reopen. */ 867 cursor.close(); 868 txn.commit(); 869 closeEnv(true); 870 openEnv(); 871 txn = env.beginTransaction(null, null); 872 cursor = db.openCursor(txn, null); 873 874 /* Position cursor to load BIN and IN. */ 875 cursor.getSearchKey(keyEntry, dataEntry, null); 876 877 /* Expect BIN and IN are still not obsolete. */ 878 assertEquals(binFile, getBINFile(cursor)); 879 assertEquals(inFile, getINFile(cursor)); 880 expectObsolete(binFile, false); 881 expectObsolete(inFile, false); 882 883 /* 884 * Add records until we move to the next BIN, so that the compressor 885 * would not need to delete the root in order to delete the BIN. 886 */ 887 if (dups) { 888 int dataVal = 1; 889 while (binFile == getBINFile(cursor)) { 890 dataVal += 1; 891 IntegerBinding.intToEntry(dataVal, dataEntry); 892 cursor.put(keyEntry, dataEntry); 893 } 894 } else { 895 int keyVal = 0; 896 while (binFile == getBINFile(cursor)) { 897 keyVal += 1; 898 IntegerBinding.intToEntry(keyVal, keyEntry); 899 cursor.put(keyEntry, dataEntry); 900 } 901 } 902 binFile = getBINFile(cursor); 903 inFile = getINFile(cursor); 904 905 /* Delete all records in the last BIN. */ 906 while (binFile == getBINFile(cursor)) { 907 cursor.delete(); 908 cursor.getLast(keyEntry, dataEntry, null); 909 } 910 911 /* Compressor daemon is not running -- they're not obsolete yet. */ 912 expectObsolete(binFile, false); 913 expectObsolete(inFile, false); 914 915 /* Close cursor and compress. */ 916 cursor.close(); 917 txn.commit(); 918 env.compress(); 919 920 /* 921 * Now expect BIN and IN to be obsolete. 922 */ 923 expectObsolete(binFile, true); 924 expectObsolete(inFile, true); 925 926 /* Close with partial checkpoint and reopen. */ 927 performPartialCheckpoint(true); // truncateUtilizationInfo 928 openEnv(); 929 930 /* 931 * Expect both files to be obsolete after recovery, because the 932 * FileSummaryLN and MapLN were written prior to the checkpoint during 933 * compression. 934 */ 935 expectObsolete(binFile, true); 936 expectObsolete(inFile, true); 937 938 /* 939 * expectAccurateObsoleteLNCount is false because the deleted LN is not 940 * counted obsolete correctly as described in RecoveryManager 941 * redoUtilizationInfo. 942 */ 943 closeEnv(true, // doCheckpoint 944 false); // expectAccurateObsoleteLNCount 945 } 946 947 /** 948 * Performs testDelete with duplicates. 949 */ 950 @Test testDeleteDup()951 public void testDeleteDup() 952 throws DatabaseException, IOException { 953 954 dups = true; 955 testDelete(); 956 } 957 958 /** 959 * Tests that truncating a database is counted correctly. 960 * Tests recovery also. 961 */ 962 @Test testTruncate()963 public void testTruncate() 964 throws DatabaseException, IOException { 965 966 /* Expect inaccurate LN sizes only if we force a tree walk. */ 967 final boolean expectAccurateObsoleteLNSize = 968 !DatabaseImpl.forceTreeWalkForTruncateAndRemove; 969 970 openAndWriteDatabase(); 971 long binFile = getBINFile(cursor); 972 long inFile = getINFile(cursor); 973 974 /* Close normally and reopen. */ 975 cursor.close(); 976 txn.commit(); 977 closeEnv(true, // doCheckpoint 978 true, // expectAccurateObsoleteLNCount 979 expectAccurateObsoleteLNSize); 980 openEnv(); 981 db.close(); 982 db = null; 983 /* Truncate. */ 984 txn = env.beginTransaction(null, null); 985 env.truncateDatabase(txn, DB_NAME, false /* returnCount */); 986 truncateOrRemoveDone = true; 987 txn.commit(); 988 989 /* 990 * Expect BIN and IN are obsolete. Do not check DbFileSummary when we 991 * truncate/remove, since the old DatabaseImpl is gone. 992 */ 993 expectObsolete(binFile, true, false /*checkDbFileSummary*/); 994 expectObsolete(inFile, true, false /*checkDbFileSummary*/); 995 996 /* Close with partial checkpoint and reopen. */ 997 performPartialCheckpoint(true, // truncateUtilizationInfo 998 true, // expectAccurateObsoleteLNCount 999 expectAccurateObsoleteLNSize); 1000 openEnv(); 1001 1002 /* Expect BIN and IN are counted obsolete during recovery. */ 1003 expectObsolete(binFile, true, false /*checkDbFileSummary*/); 1004 expectObsolete(inFile, true, false /*checkDbFileSummary*/); 1005 1006 /* 1007 * expectAccurateObsoleteLNSize is false because the size of the 1008 * deleted NameLN is not counted during recovery, as with other 1009 * abortLsns as described in RecoveryManager redoUtilizationInfo. 1010 */ 1011 closeEnv(true, // doCheckpoint 1012 true, // expectAccurateObsoleteLNCount 1013 false); // expectAccurateObsoleteLNSize 1014 } 1015 1016 /** 1017 * Tests that truncating a database is counted correctly. 1018 * Tests recovery also. 1019 */ 1020 @Test testRemove()1021 public void testRemove() 1022 throws DatabaseException, IOException { 1023 1024 /* Expect inaccurate LN sizes only if we force a tree walk. */ 1025 final boolean expectAccurateObsoleteLNSize = 1026 !DatabaseImpl.forceTreeWalkForTruncateAndRemove; 1027 1028 openAndWriteDatabase(); 1029 long binFile = getBINFile(cursor); 1030 long inFile = getINFile(cursor); 1031 1032 /* Close normally and reopen. */ 1033 cursor.close(); 1034 txn.commit(); 1035 closeEnv(true, // doCheckpoint 1036 true, // expectAccurateObsoleteLNCount 1037 expectAccurateObsoleteLNSize); 1038 openEnv(); 1039 1040 /* Remove. */ 1041 db.close(); 1042 db = null; 1043 txn = env.beginTransaction(null, null); 1044 env.removeDatabase(txn, DB_NAME); 1045 truncateOrRemoveDone = true; 1046 txn.commit(); 1047 1048 /* 1049 * Expect BIN and IN are obsolete. Do not check DbFileSummary when we 1050 * truncate/remove, since the old DatabaseImpl is gone. 1051 */ 1052 expectObsolete(binFile, true, false /*checkDbFileSummary*/); 1053 expectObsolete(inFile, true, false /*checkDbFileSummary*/); 1054 1055 /* Close with partial checkpoint and reopen. */ 1056 performPartialCheckpoint(true, // truncateUtilizationInfo 1057 true, // expectAccurateObsoleteLNCount 1058 expectAccurateObsoleteLNSize); 1059 openEnv(); 1060 1061 /* Expect BIN and IN are counted obsolete during recovery. */ 1062 expectObsolete(binFile, true, false /*checkDbFileSummary*/); 1063 expectObsolete(inFile, true, false /*checkDbFileSummary*/); 1064 1065 /* 1066 * expectAccurateObsoleteLNCount is false because the deleted NameLN is 1067 * not counted obsolete correctly as described in RecoveryManager 1068 * redoUtilizationInfo. 1069 */ 1070 closeEnv(true, // doCheckpoint 1071 false); // expectAccurateObsoleteLNCount 1072 } 1073 1074 /* 1075 * The xxxForceTreeWalk tests set the DatabaseImpl 1076 * forceTreeWalkForTruncateAndRemove field to true, which will force a walk 1077 * of the tree to count utilization during truncate/remove, rather than 1078 * using the per-database info. This is used to test the "old technique" 1079 * for counting utilization, which is now used only if the database was 1080 * created prior to log version 6. 1081 */ 1082 1083 @Test testTruncateForceTreeWalk()1084 public void testTruncateForceTreeWalk() 1085 throws Exception { 1086 1087 DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; 1088 try { 1089 testTruncate(); 1090 } finally { 1091 DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; 1092 } 1093 } 1094 1095 @Test testRemoveForceTreeWalk()1096 public void testRemoveForceTreeWalk() 1097 throws Exception { 1098 1099 DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; 1100 try { 1101 testRemove(); 1102 } finally { 1103 DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; 1104 } 1105 } 1106 expectObsolete(long file, boolean obsolete)1107 private void expectObsolete(long file, boolean obsolete) { 1108 expectObsolete(file, obsolete, true /*checkDbFileSummary*/); 1109 } 1110 expectObsolete(long file, boolean obsolete, boolean checkDbFileSummary)1111 private void expectObsolete(long file, 1112 boolean obsolete, 1113 boolean checkDbFileSummary) { 1114 FileSummary fileSummary = getFileSummary(file); 1115 assertEquals("totalINCount", 1116 1, fileSummary.totalINCount); 1117 assertEquals("obsoleteINCount", 1118 obsolete ? 1 : 0, fileSummary.obsoleteINCount); 1119 1120 if (checkDbFileSummary) { 1121 DbFileSummary dbFileSummary = getDbFileSummary(file); 1122 assertEquals("db totalINCount", 1123 1, dbFileSummary.totalINCount); 1124 assertEquals("db obsoleteINCount", 1125 obsolete ? 1 : 0, dbFileSummary.obsoleteINCount); 1126 } 1127 } 1128 expectObsolete(long file, int obsoleteCount)1129 private void expectObsolete(long file, int obsoleteCount) { 1130 FileSummary fileSummary = getFileSummary(file); 1131 assertEquals("totalINCount", 1132 1, fileSummary.totalINCount); 1133 assertEquals("obsoleteINCount", 1134 obsoleteCount, fileSummary.obsoleteINCount); 1135 1136 DbFileSummary dbFileSummary = getDbFileSummary(file); 1137 assertEquals("db totalINCount", 1138 1, dbFileSummary.totalINCount); 1139 assertEquals("db obsoleteINCount", 1140 obsoleteCount, dbFileSummary.obsoleteINCount); 1141 } 1142 getINFile(Cursor cursor)1143 private long getINFile(Cursor cursor) 1144 throws DatabaseException { 1145 1146 IN in = TestUtils.getIN(TestUtils.getBIN(cursor)); 1147 long lsn = in.getLastLoggedVersion(); 1148 assertTrue(lsn != DbLsn.NULL_LSN); 1149 return DbLsn.getFileNumber(lsn); 1150 } 1151 getBINFile(Cursor cursor)1152 private long getBINFile(Cursor cursor) { 1153 long lsn = TestUtils.getBIN(cursor).getLastLoggedVersion(); 1154 assertTrue(lsn != DbLsn.NULL_LSN); 1155 return DbLsn.getFileNumber(lsn); 1156 } 1157 getFullBINFile(Cursor cursor)1158 private long getFullBINFile(Cursor cursor) { 1159 long lsn = TestUtils.getBIN(cursor).getLastFullVersion(); 1160 assertTrue(lsn != DbLsn.NULL_LSN); 1161 return DbLsn.getFileNumber(lsn); 1162 } 1163 getDeltaBINFile(Cursor cursor)1164 private long getDeltaBINFile(Cursor cursor) { 1165 long lsn = TestUtils.getBIN(cursor).getLastDeltaVersion(); 1166 if (lsn == DbLsn.NULL_LSN) { 1167 return -1; 1168 } 1169 return DbLsn.getFileNumber(lsn); 1170 } 1171 1172 /** 1173 * Returns the utilization summary for a given log file. 1174 */ getFileSummary(long file)1175 private FileSummary getFileSummary(long file) { 1176 return envImpl.getUtilizationProfile() 1177 .getFileSummaryMap(true) 1178 .get(new Long(file)); 1179 } 1180 1181 /** 1182 * Returns the per-database utilization summary for a given log file. 1183 */ getDbFileSummary(long file)1184 private DbFileSummary getDbFileSummary(long file) { 1185 return dbImpl.getDbFileSummary 1186 (new Long(file), false /*willModify*/); 1187 } 1188 performPartialCheckpoint(boolean truncateUtilizationInfo)1189 private void performPartialCheckpoint(boolean truncateUtilizationInfo) 1190 throws DatabaseException, IOException { 1191 1192 performPartialCheckpoint(truncateUtilizationInfo, 1193 true, // expectAccurateObsoleteLNCount 1194 true); // expectAccurateObsoleteLNSize 1195 } 1196 performPartialCheckpoint(boolean truncateUtilizationInfo, boolean expectAccurateObsoleteLNCount)1197 private void performPartialCheckpoint(boolean truncateUtilizationInfo, 1198 boolean 1199 expectAccurateObsoleteLNCount) 1200 throws DatabaseException, IOException { 1201 1202 performPartialCheckpoint(truncateUtilizationInfo, 1203 expectAccurateObsoleteLNCount, 1204 expectAccurateObsoleteLNCount); 1205 } 1206 1207 /** 1208 * Performs a checkpoint and truncates the log before the last CkptEnd. If 1209 * truncateUtilizationInfo is true, truncates before the FileSummaryLNs 1210 * that appear at the end of the checkpoint. The environment should be 1211 * open when this method is called, and it will be closed when it returns. 1212 */ performPartialCheckpoint(boolean truncateUtilizationInfo, boolean expectAccurateObsoleteLNCount, boolean expectAccurateObsoleteLNSize)1213 private void performPartialCheckpoint 1214 (boolean truncateUtilizationInfo, 1215 boolean expectAccurateObsoleteLNCount, 1216 boolean expectAccurateObsoleteLNSize) 1217 throws DatabaseException, IOException { 1218 1219 /* Do a normal checkpoint. */ 1220 env.checkpoint(forceConfig); 1221 long eofLsn = envImpl.getFileManager().getNextLsn(); 1222 long lastLsn = envImpl.getFileManager().getLastUsedLsn(); 1223 1224 /* Searching backward from end, find last CkptEnd. */ 1225 SearchFileReader searcher = 1226 new SearchFileReader(envImpl, 1000, false, lastLsn, eofLsn, 1227 LogEntryType.LOG_CKPT_END); 1228 assertTrue(searcher.readNextEntry()); 1229 long ckptEnd = searcher.getLastLsn(); 1230 long truncateLsn = ckptEnd; 1231 1232 if (truncateUtilizationInfo) { 1233 1234 /* Searching backward from CkptEnd, find last CkptStart. */ 1235 searcher = 1236 new SearchFileReader(envImpl, 1000, false, ckptEnd, eofLsn, 1237 LogEntryType.LOG_CKPT_START); 1238 assertTrue(searcher.readNextEntry()); 1239 long ckptStart = searcher.getLastLsn(); 1240 1241 /* 1242 * Searching forward from CkptStart, find first MapLN for a user 1243 * database (ID GT 2), or if none, the last MapLN for a non-user 1244 * database. MapLNs are written after writing root INs and before 1245 * all FileSummaryLNs. This will find the position at which to 1246 * truncate all MapLNs and FileSummaryLNs, but not INs below the 1247 * mapping tree. 1248 */ 1249 searcher = new SearchFileReader(envImpl, 1000, true, 1250 ckptStart, eofLsn, 1251 LogEntryType.LOG_MAPLN); 1252 while (searcher.readNextEntry()) { 1253 MapLN mapLN = (MapLN) searcher.getLastObject(); 1254 truncateLsn = searcher.getLastLsn(); 1255 if (mapLN.getDatabase().getId().getId() > 2) { 1256 break; 1257 } 1258 } 1259 } 1260 1261 /* 1262 * Close without another checkpoint, although it doesn't matter since 1263 * we would truncate before it. 1264 */ 1265 closeEnv(false, // doCheckpoint 1266 expectAccurateObsoleteLNCount, 1267 expectAccurateObsoleteLNSize); 1268 1269 /* Truncate the log. */ 1270 EnvironmentConfig envConfig = new EnvironmentConfig(); 1271 envConfig.setAllowCreate(true); 1272 envConfig.setConfigParam 1273 (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); 1274 envConfig.setConfigParam 1275 (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false"); 1276 envConfig.setConfigParam 1277 (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); 1278 envConfig.setConfigParam 1279 (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false"); 1280 if (envMultiSubDir) { 1281 envConfig.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, 1282 DATA_DIRS + ""); 1283 } 1284 EnvironmentImpl cmdEnv = 1285 DbInternal.getEnvironmentImpl(new Environment(envHome, envConfig)); 1286 cmdEnv.getFileManager().truncateLog(DbLsn.getFileNumber(truncateLsn), 1287 DbLsn.getFileOffset(truncateLsn)); 1288 cmdEnv.abnormalClose(); 1289 } 1290 } 1291