1 /*- 2 * See the file LICENSE for redistribution information. 3 * 4 * Copyright (c) 2002, 2014 Oracle and/or its affiliates. All rights reserved. 5 * 6 */ 7 8 package com.sleepycat.je.evictor; 9 10 import static org.junit.Assert.assertEquals; 11 import static org.junit.Assert.assertSame; 12 import static org.junit.Assert.assertTrue; 13 import static org.junit.Assert.fail; 14 15 import java.io.File; 16 17 import org.junit.After; 18 import org.junit.Before; 19 import org.junit.Test; 20 21 import com.sleepycat.bind.tuple.IntegerBinding; 22 import com.sleepycat.je.CacheMode; 23 import com.sleepycat.je.CheckpointConfig; 24 import com.sleepycat.je.Cursor; 25 import com.sleepycat.je.Database; 26 import com.sleepycat.je.DatabaseConfig; 27 import com.sleepycat.je.DatabaseEntry; 28 import com.sleepycat.je.DatabaseException; 29 import com.sleepycat.je.DbInternal; 30 import com.sleepycat.je.Environment; 31 import com.sleepycat.je.EnvironmentConfig; 32 import com.sleepycat.je.EnvironmentMutableConfig; 33 import com.sleepycat.je.EnvironmentStats; 34 import com.sleepycat.je.LockMode; 35 import com.sleepycat.je.OperationStatus; 36 import com.sleepycat.je.Transaction; 37 import com.sleepycat.je.config.EnvironmentParams; 38 import com.sleepycat.je.dbi.DatabaseImpl; 39 import com.sleepycat.je.dbi.DbTree; 40 import com.sleepycat.je.dbi.EnvironmentImpl; 41 import com.sleepycat.je.dbi.MemoryBudget; 42 import com.sleepycat.je.junit.JUnitThread; 43 import com.sleepycat.je.tree.IN; 44 import com.sleepycat.je.txn.Txn; 45 import com.sleepycat.je.util.TestUtils; 46 import com.sleepycat.util.test.SharedTestUtils; 47 import com.sleepycat.util.test.TestBase; 48 49 /** 50 * This tests exercises the act of eviction and determines whether the 51 * expected nodes have been evicted properly. 52 */ 53 public class EvictActionTest extends TestBase { 54 55 private static final boolean DEBUG = false; 56 private static final int NUM_KEYS = 60; 57 private static final int NUM_DUPS = 50; 58 private static final int BIG_CACHE_SIZE = 500000; 59 private static final int SMALL_CACHE_SIZE = (int) 60 MemoryBudget.MIN_MAX_MEMORY_SIZE; 61 62 private File envHome = null; 63 private Environment env = null; 64 private Database db = null; 65 private int actualLNs = 0; 66 private int actualINs = 0; 67 EvictActionTest()68 public EvictActionTest() { 69 envHome = SharedTestUtils.getTestDir(); 70 } 71 72 @Before setUp()73 public void setUp() 74 throws Exception { 75 76 IN.ACCUMULATED_LIMIT = 0; 77 Txn.ACCUMULATED_LIMIT = 0; 78 super.setUp(); 79 } 80 81 @After tearDown()82 public void tearDown() { 83 84 if (env != null) { 85 try { 86 env.close(); 87 } catch (Throwable e) { 88 System.out.println("tearDown: " + e); 89 } 90 } 91 envHome = null; 92 env = null; 93 db = null; 94 } 95 96 @Test testEvict()97 public void testEvict() 98 throws Throwable { 99 100 doEvict(50, SMALL_CACHE_SIZE, true); 101 } 102 103 @Test testNoNeedToEvict()104 public void testNoNeedToEvict() 105 throws Throwable { 106 107 doEvict(80, BIG_CACHE_SIZE, false); 108 } 109 110 /** 111 * Evict in very controlled circumstances. Check that we first strip 112 * BINs and later evict BINS. 113 */ doEvict(int floor, int maxMem, boolean shouldEvict)114 private void doEvict(int floor, 115 int maxMem, 116 boolean shouldEvict) 117 throws Throwable { 118 119 openEnv(floor, maxMem); 120 insertData(NUM_KEYS); 121 122 /* Evict once after insert. */ 123 evictAndCheck(shouldEvict, NUM_KEYS); 124 125 /* Evict again after verification. */ 126 evictAndCheck(shouldEvict, NUM_KEYS); 127 128 closeEnv(); 129 } 130 131 @Test testSetCacheSize()132 public void testSetCacheSize() 133 throws DatabaseException { 134 135 /* Start with large cache size. */ 136 openEnv(80, BIG_CACHE_SIZE); 137 EnvironmentMutableConfig config = env.getMutableConfig(); 138 insertData(NUM_KEYS); 139 140 /* No need to evict. */ 141 verifyData(NUM_KEYS); 142 evictAndCheck(false, NUM_KEYS); 143 144 /* Set small cache size. */ 145 config.setCacheSize(SMALL_CACHE_SIZE); 146 env.setMutableConfig(config); 147 148 /* Expect eviction. */ 149 verifyData(NUM_KEYS); 150 evictAndCheck(true, NUM_KEYS); 151 152 /* Set large cache size. */ 153 config.setCacheSize(BIG_CACHE_SIZE); 154 env.setMutableConfig(config); 155 156 /* Expect no eviction. */ 157 verifyData(NUM_KEYS); 158 evictAndCheck(false, NUM_KEYS); 159 160 closeEnv(); 161 } 162 163 @Test testSetCachePercent()164 public void testSetCachePercent() 165 throws DatabaseException { 166 167 int nKeys = NUM_KEYS * 500; 168 169 /* Start with large cache size. */ 170 openEnv(80, BIG_CACHE_SIZE); 171 EnvironmentMutableConfig config = env.getMutableConfig(); 172 config.setCacheSize(0); 173 config.setCachePercent(90); 174 env.setMutableConfig(config); 175 insertData(nKeys); 176 177 /* No need to evict. */ 178 verifyData(nKeys); 179 evictAndCheck(false, nKeys); 180 181 /* Set small cache percent. */ 182 config.setCacheSize(0); 183 config.setCachePercent(1); 184 env.setMutableConfig(config); 185 186 /* Expect eviction. */ 187 verifyData(nKeys); 188 evictAndCheck(true, nKeys); 189 190 /* Set large cache percent. */ 191 config.setCacheSize(0); 192 config.setCachePercent(90); 193 env.setMutableConfig(config); 194 195 /* Expect no eviction. */ 196 verifyData(nKeys); 197 evictAndCheck(false, nKeys); 198 199 closeEnv(); 200 } 201 202 @Test testThreadedCacheSizeChanges()203 public void testThreadedCacheSizeChanges() 204 throws DatabaseException { 205 206 final int N_ITERS = 10; 207 openEnv(80, BIG_CACHE_SIZE); 208 insertData(NUM_KEYS); 209 210 JUnitThread writer = new JUnitThread("Writer") { 211 @Override 212 public void testBody() 213 throws DatabaseException { 214 for (int i = 0; i < N_ITERS; i += 1) { 215 env.evictMemory(); 216 /* insertData will update if data exists. */ 217 insertData(NUM_KEYS); 218 env.evictMemory(); 219 EnvironmentMutableConfig config = env.getMutableConfig(); 220 config.setCacheSize(SMALL_CACHE_SIZE); 221 env.setMutableConfig(config); 222 } 223 } 224 }; 225 226 JUnitThread reader = new JUnitThread("Reader") { 227 @Override 228 public void testBody() 229 throws DatabaseException { 230 for (int i = 0; i < N_ITERS; i += 1) { 231 env.evictMemory(); 232 verifyData(NUM_KEYS); 233 env.evictMemory(); 234 EnvironmentMutableConfig config = env.getMutableConfig(); 235 config.setCacheSize(BIG_CACHE_SIZE); 236 env.setMutableConfig(config); 237 } 238 } 239 }; 240 241 writer.start(); 242 reader.start(); 243 244 try { 245 writer.finishTest(); 246 } catch (Throwable e) { 247 try { 248 reader.finishTest(); 249 } catch (Throwable ignore) { } 250 e.printStackTrace(); 251 fail(e.toString()); 252 } 253 254 try { 255 reader.finishTest(); 256 } catch (Throwable e) { 257 e.printStackTrace(); 258 fail(e.toString()); 259 } 260 261 closeEnv(); 262 } 263 264 @Test testSmallCacheSettings()265 public void testSmallCacheSettings() 266 throws DatabaseException { 267 268 /* 269 * With a cache size > 600 KB, the min tree usage should be the default 270 * value. 271 */ 272 openEnv(0, 1200 * 1024); 273 EnvironmentMutableConfig config = env.getMutableConfig(); 274 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 275 MemoryBudget mb = envImpl.getMemoryBudget(); 276 assertEquals(500 * 1024, mb.getMinTreeMemoryUsage()); 277 278 /* 279 * With a cache size > 1000 KB, evict bytes may be > 500 KB but we 280 * should not evict over half the cache size. 281 */ 282 putLargeData(1200, 1024); 283 284 EnvironmentStats stats = env.getStats(null); 285 286 env.evictMemory(); 287 stats = env.getStats(null); 288 assertTrue(stats.getCacheTotalBytes() >= 1200 * 1024 / 2); 289 290 /* 291 * With a cache size of 500 KB, the min tree usage should be the amount 292 * available in the cache after the buffer bytes are subtracted. 293 */ 294 config.setCacheSize(500 * 1024); 295 env.setMutableConfig(config); 296 stats = env.getStats(null); 297 assertEquals(500 * 1024 - stats.getBufferBytes(), 298 mb.getMinTreeMemoryUsage()); 299 300 /* 301 * With a cache size of 500 KB, evict bytes may be < 500 KB but we 302 * should not evict over half the cache size. 303 */ 304 putLargeData(500, 1024); 305 env.evictMemory(); 306 stats = env.getStats(null); 307 assertTrue(stats.getCacheTotalBytes() >= 500 * 1024 / 2); 308 309 /* 310 * Even when using a large amount of non-tree memory, the tree memory 311 * usage should not go below the minimum. 312 */ 313 mb.updateAdminMemoryUsage(500 * 1024); 314 env.evictMemory(); 315 stats = env.getStats(null); 316 long treeBytes = stats.getDataBytes() + 317 50 * 1024 /* larger than any LN or IN */; 318 assertTrue(treeBytes >= mb.getMinTreeMemoryUsage()); 319 mb.updateAdminMemoryUsage(-(500 * 1024)); 320 321 /* Allow changing the min tree usage explicitly. */ 322 config.setCacheSize(500 * 1024); 323 config.setConfigParam("je.tree.minMemory", String.valueOf(50 * 1024)); 324 env.setMutableConfig(config); 325 assertEquals(50 * 1024, mb.getMinTreeMemoryUsage()); 326 327 /* The min tree usage may not be larger than the cache. */ 328 config.setCacheSize(500 * 1024); 329 config.setConfigParam("je.tree.minMemory", String.valueOf(900 * 1024)); 330 env.setMutableConfig(config); 331 stats = env.getStats(null); 332 assertEquals(500 * 1024 - stats.getBufferBytes(), 333 mb.getMinTreeMemoryUsage()); 334 335 closeEnv(); 336 } 337 338 /** 339 * We now allow eviction of the root IN of a DB, whether the DB is closed 340 * or not. Check that basic root eviction works. [#13415] 341 */ 342 @Test testRootINEviction()343 public void testRootINEviction() 344 throws DatabaseException { 345 346 DatabaseEntry entry = new DatabaseEntry(new byte[1]); 347 OperationStatus status; 348 349 openEnv(80, SMALL_CACHE_SIZE); 350 351 DatabaseConfig dbConfig = new DatabaseConfig(); 352 dbConfig.setAllowCreate(true); 353 Database db1 = env.openDatabase(null, "db1", dbConfig); 354 355 /* Root starts out null. */ 356 assertTrue(!isRootResident(db1)); 357 /* It is created when we insert the first record. */ 358 status = db1.put(null, entry, entry); 359 assertSame(OperationStatus.SUCCESS, status); 360 assertTrue(isRootResident(db1)); 361 /* It is evicted when necessary. */ 362 forceEviction(); 363 assertTrue(!isRootResident(db1)); 364 /* And fetched again when needed. */ 365 status = db1.get(null, entry, entry, null); 366 assertSame(OperationStatus.SUCCESS, status); 367 assertTrue(isRootResident(db1)); 368 369 /* Deferred write DBs work in the same way. */ 370 dbConfig.setDeferredWrite(true); 371 Database db2 = env.openDatabase(null, "db2", dbConfig); 372 status = db2.put(null, entry, entry); 373 assertSame(OperationStatus.SUCCESS, status); 374 assertTrue(isRootResident(db2)); 375 /* It is evicted when necessary. */ 376 forceEviction(); 377 assertTrue(!isRootResident(db2)); 378 /* And fetched again when needed. */ 379 status = db2.get(null, entry, entry, null); 380 assertSame(OperationStatus.SUCCESS, status); 381 assertTrue(isRootResident(db2)); 382 /* Deferred-write eviction also works when the root is not dirty. */ 383 db2.sync(); 384 forceEviction(); 385 assertTrue(!isRootResident(db2)); 386 387 db2.close(); 388 db1.close(); 389 closeEnv(); 390 } 391 392 /** 393 * We now allow eviction of the MapLN and higher level INs in the DB mappng 394 * tree when DBs are closed. Check that basic mapping tree IN eviction 395 * works. [#13415] 396 */ 397 @Test testMappingTreeEviction()398 public void testMappingTreeEviction() 399 throws DatabaseException { 400 401 DatabaseConfig dbConfig = new DatabaseConfig(); 402 dbConfig.setAllowCreate(true); 403 404 DatabaseEntry entry = new DatabaseEntry(new byte[1]); 405 OperationStatus status; 406 407 openEnv(80, SMALL_CACHE_SIZE); 408 409 /* Baseline mapping tree LNs and INs. */ 410 final int baseLNs = 2; // Utilization DB and test DB 411 final int baseINs = 2; // Root IN and BIN 412 checkMappingTree(baseLNs, baseINs); 413 forceEviction(); 414 checkMappingTree(baseLNs, baseINs); 415 416 /* 417 * Create enough DBs to fill up a BIN in the mapping DB. NODE_MAX is 418 * configured to be 4 in this test. There are already 2 DBs open. 419 */ 420 final int nDbs = 4; 421 Database[] dbs = new Database[nDbs]; 422 for (int i = 0; i < nDbs; i += 1) { 423 dbs[i] = env.openDatabase(null, "db" + i, dbConfig); 424 status = dbs[i].put(null, entry, entry); 425 assertSame(OperationStatus.SUCCESS, status); 426 assertTrue(isRootResident(dbs[i])); 427 } 428 final int openLNs = baseLNs + nDbs; // Add 1 MapLN per open DB 429 final int openINs = baseINs + 1; // Add 1 BIN in the mapping tree 430 checkMappingTree(openLNs, openINs); 431 forceEviction(); 432 checkMappingTree(openLNs, openINs); 433 434 /* Close DBs and force eviction. */ 435 for (int i = 0; i < nDbs; i += 1) { 436 dbs[i].close(); 437 } 438 forceEviction(); 439 checkMappingTree(baseLNs, baseINs); 440 441 /* Re-open the DBs, opening each DB twice. */ 442 Database[] dbs2 = new Database[nDbs]; 443 for (int i = 0; i < nDbs; i += 1) { 444 dbs[i] = env.openDatabase(null, "db" + i, dbConfig); 445 dbs2[i] = env.openDatabase(null, "db" + i, dbConfig); 446 } 447 checkMappingTree(openLNs, openINs); 448 forceEviction(); 449 checkMappingTree(openLNs, openINs); 450 451 /* Close one handle only, MapLN eviction should not occur. */ 452 for (int i = 0; i < nDbs; i += 1) { 453 dbs[i].close(); 454 } 455 forceEviction(); 456 checkMappingTree(openLNs, openINs); 457 458 /* Close the other handles, eviction should occur. */ 459 for (int i = 0; i < nDbs; i += 1) { 460 dbs2[i].close(); 461 } 462 forceEviction(); 463 checkMappingTree(baseLNs, baseINs); 464 465 closeEnv(); 466 } 467 468 /** 469 * Checks that cleaning performs memory budget calculations correctly for 470 * evicted databases (non-resident MapLNs). [#21686] 471 */ testCleaningAfterMappingTreeEviction()472 public void testCleaningAfterMappingTreeEviction() 473 throws DatabaseException { 474 475 DatabaseConfig dbConfig = new DatabaseConfig(); 476 dbConfig.setAllowCreate(true); 477 478 DatabaseEntry key = new DatabaseEntry(new byte[1000]); 479 DatabaseEntry data = new DatabaseEntry(new byte[10000]); 480 OperationStatus status; 481 482 EnvironmentConfig envConfig = 483 getEnvConfig(0, SMALL_CACHE_SIZE, false /*readonly*/); 484 envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, 485 String.valueOf(1024 * 1024)); 486 envConfig.setConfigParam(EnvironmentConfig.CLEANER_MIN_UTILIZATION, 487 "75"); 488 openEnv(envConfig); 489 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 490 491 /* 492 * Create enough DBs to fill several 10 or more files and close them so 493 * they'll be evicted below. 494 */ 495 final int nDbs = 1000; 496 for (int i = 0; i < nDbs; i += 1) { 497 Database db = env.openDatabase(null, "db" + i, dbConfig); 498 status = db.put(null, key, data); 499 assertSame(OperationStatus.SUCCESS, status); 500 status = db.delete(null, key); 501 assertSame(OperationStatus.SUCCESS, status); 502 db.close(); 503 } 504 env.checkpoint(new CheckpointConfig().setForce(true)); 505 forceEviction(); 506 507 /* 508 * Clean and checkpoint repeatedly to create a scenario where a 509 * non-resident MapLN is migrated by the cleaner [#21686]. With only a 510 * single call to cleanLog, the MapLN will be resident due to the 511 * nearby LNs for that database, since the cleaner does read-ahead 512 * during LN processing. 513 * 514 * Prior to the bug fix, TestUtils.validateNodeMemUsage (called by 515 * forceEviction below) would report a mismatch in the tree admin 516 * memory usage. 517 */ 518 for (int i = 0; i < 10; i += 1) { 519 env.cleanLog(); 520 env.checkpoint(new CheckpointConfig().setForce(true)); 521 forceEviction(); 522 } 523 524 /* Final check for good measure. */ 525 TestUtils.validateNodeMemUsage(envImpl, true); 526 527 closeEnv(); 528 } 529 530 /** 531 * Checks that a dirty root IN is not evicted in a read-only environment. 532 * [#16368] 533 */ 534 @Test testReadOnlyRootINEviction()535 public void testReadOnlyRootINEviction() 536 throws DatabaseException { 537 538 OperationStatus status; 539 540 openEnv(80, SMALL_CACHE_SIZE); 541 542 /* Big record will be used to force eviction. */ 543 DatabaseEntry bigRecordKey = new DatabaseEntry(new byte[1]); 544 status = db.put(null, bigRecordKey, 545 new DatabaseEntry(new byte[BIG_CACHE_SIZE])); 546 assertSame(OperationStatus.SUCCESS, status); 547 548 /* Open DB1 and insert a record to create the root IN. */ 549 DatabaseConfig dbConfig = new DatabaseConfig(); 550 dbConfig.setAllowCreate(true); 551 Database db1 = env.openDatabase(null, "db1", dbConfig); 552 553 DatabaseEntry smallRecordKey = new DatabaseEntry(new byte[1]); 554 DatabaseEntry smallData = new DatabaseEntry(new byte[1]); 555 status = db1.put(null, smallRecordKey, smallData); 556 assertSame(OperationStatus.SUCCESS, status); 557 558 /* Close environment and re-open it read-only. */ 559 db1.close(); 560 closeEnv(); 561 562 EnvironmentConfig envConfig = 563 getEnvConfig(80, SMALL_CACHE_SIZE, true /*readOnly*/); 564 envConfig.setConfigParam 565 (EnvironmentParams.EVICTOR_NODES_PER_SCAN.getName(), "1"); 566 openEnv(envConfig); 567 568 dbConfig.setReadOnly(true); 569 dbConfig.setAllowCreate(false); 570 db1 = env.openDatabase(null, "db1", dbConfig); 571 572 /* Load a record to load the root IN. */ 573 status = db1.get(null, smallRecordKey, new DatabaseEntry(), null); 574 assertSame(OperationStatus.SUCCESS, status); 575 assertTrue(isRootResident(db1)); 576 577 /* 578 * Set the root dirty to prevent eviction. In real life, this can only 579 * be done by recovery in a read-only environment, but that's very 580 * difficult to simulate precisely. 581 */ 582 IN rootIN = DbInternal.getDatabaseImpl(db1). 583 getTree(). 584 getRootIN(CacheMode.DEFAULT); 585 rootIN.setDirty(true); 586 rootIN.releaseLatch(); 587 588 /* Root should not be evicted while dirty. */ 589 forceReadOnlyEviction(bigRecordKey); 590 assertTrue(isRootResident(db1)); 591 forceReadOnlyEviction(bigRecordKey); 592 assertTrue(isRootResident(db1)); 593 594 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 595 Evictor evictor = envImpl.getEvictor(); 596 597 /* When made non-dirty, it can be evicted. */ 598 rootIN.setDirty(false); 599 evictor.addBack(rootIN); 600 601 forceReadOnlyEviction(bigRecordKey); 602 assertTrue(!isRootResident(db1)); 603 604 db1.close(); 605 closeEnv(); 606 } 607 608 /** 609 * Check that opening a database in a transaction and then aborting the 610 * transaction will decrement the database use count. [#13415] 611 */ 612 @Test testAbortOpen()613 public void testAbortOpen() 614 throws DatabaseException { 615 616 EnvironmentConfig envConfig = TestUtils.initEnvConfig(); 617 envConfig.setAllowCreate(true); 618 envConfig.setTransactional(true); 619 envConfig.setConfigParam(EnvironmentParams. 620 ENV_DB_EVICTION.getName(), "true"); 621 env = new Environment(envHome, envConfig); 622 623 /* Abort the txn used to open a database. */ 624 Transaction txn = env.beginTransaction(null, null); 625 DatabaseConfig dbConfig = new DatabaseConfig(); 626 dbConfig.setAllowCreate(true); 627 dbConfig.setTransactional(true); 628 Database db1 = env.openDatabase(txn, "db1", dbConfig); 629 DatabaseImpl saveDb = DbInternal.getDatabaseImpl(db1); 630 txn.abort(); 631 632 /* DB should not be in use and does not have to be closed. */ 633 assertEquals(false, saveDb.isInUse()); 634 635 /* 636 * Environment.close will not throw an exception, even though the DB 637 * has not been closed. The abort took care of cleaning up the handle. 638 */ 639 closeEnv(); 640 641 /* 642 * Try a non-transactional DB open that throws an exception because we 643 * create it exclusively and it already exists. The use count should 644 * be decremented. 645 */ 646 env = new Environment(envHome, envConfig); 647 dbConfig.setAllowCreate(true); 648 dbConfig.setExclusiveCreate(true); 649 dbConfig.setTransactional(false); 650 db1 = env.openDatabase(null, "db1", dbConfig); 651 saveDb = DbInternal.getDatabaseImpl(db1); 652 try { 653 env.openDatabase(null, "db1", dbConfig); 654 fail(); 655 } catch (DatabaseException e) { 656 assertTrue(e.getMessage().indexOf("already exists") >= 0); 657 } 658 db1.close(); 659 assertEquals(false, saveDb.isInUse()); 660 661 /* 662 * Try a non-transactional DB open that throws an exception because we 663 * change the duplicatesAllowed setting. The use count should be 664 * decremented. 665 */ 666 dbConfig.setSortedDuplicates(true); 667 dbConfig.setExclusiveCreate(false); 668 try { 669 env.openDatabase(null, "db1", dbConfig); 670 fail(); 671 } catch (IllegalArgumentException e) { 672 assertTrue(e.getMessage().indexOf("sortedDuplicates") >= 0); 673 } 674 assertEquals(false, saveDb.isInUse()); 675 676 closeEnv(); 677 } 678 679 /** 680 * Check for the expected number of nodes in the mapping DB. 681 */ checkMappingTree(int expectLNs, int expectINs)682 private void checkMappingTree(int expectLNs, int expectINs) 683 throws DatabaseException { 684 685 IN root = DbInternal.getEnvironmentImpl(env). 686 getDbTree().getDb(DbTree.ID_DB_ID).getTree(). 687 getRootIN(CacheMode.UNCHANGED); 688 actualLNs = 0; 689 actualINs = 0; 690 countMappingTree(root); 691 root.releaseLatch(); 692 assertEquals("LNs", expectLNs, actualLNs); 693 assertEquals("INs", expectINs, actualINs); 694 } 695 countMappingTree(IN parent)696 private void countMappingTree(IN parent) { 697 actualINs += 1; 698 for (int i = 0; i < parent.getNEntries(); i += 1) { 699 if (parent.getTarget(i) != null) { 700 if (parent.getTarget(i) instanceof IN) { 701 countMappingTree((IN) parent.getTarget(i)); 702 } else { 703 actualLNs += 1; 704 } 705 } 706 } 707 } 708 709 /** 710 * Returns whether the root IN is currently resident for the given DB. 711 */ isRootResident(Database dbParam)712 private boolean isRootResident(Database dbParam) { 713 return DbInternal.getDatabaseImpl(dbParam) 714 .getTree() 715 .isRootResident(); 716 } 717 718 /** 719 * Force eviction by inserting a large record in the pre-opened DB. 720 */ forceEviction()721 private void forceEviction() 722 throws DatabaseException { 723 724 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 725 OperationStatus status; 726 727 /* 728 * Repeat twice to cause a 2nd pass over the INList. The second pass 729 * evicts BINs that were only stripped of LNs in the first pass. 730 */ 731 for (int i = 0; i < 2; i += 1) { 732 Cursor c = db.openCursor(null, null); 733 status = c.put(new DatabaseEntry(new byte[1]), 734 new DatabaseEntry(new byte[BIG_CACHE_SIZE])); 735 assertSame(OperationStatus.SUCCESS, status); 736 737 /* 738 * Evict while cursor pins LN memory, to ensure eviction of other 739 * DB INs, including the DB root. When lruOnly=false, root IN 740 * eviction may not occur unless a cursor is used to pin the LN. 741 */ 742 env.evictMemory(); 743 744 status = c.delete(); 745 assertSame(OperationStatus.SUCCESS, status); 746 747 c.close(); 748 } 749 750 TestUtils.validateNodeMemUsage(envImpl, true); 751 } 752 753 /** 754 * Force eviction by reading a large record. 755 */ forceReadOnlyEviction(DatabaseEntry key)756 private void forceReadOnlyEviction(DatabaseEntry key) 757 throws DatabaseException { 758 759 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 760 OperationStatus status; 761 762 /* 763 * Repeat twice to cause a 2nd pass over the INList. The second pass 764 * evicts BINs that were only stripped of LNs in the first pass. 765 */ 766 for (int i = 0; i < 2; i += 1) { 767 Cursor c = db.openCursor(null, null); 768 status = c.getSearchKey(key, new DatabaseEntry(), null); 769 assertSame(OperationStatus.SUCCESS, status); 770 771 /* 772 * Evict while cursor pins LN memory, to ensure eviction of other 773 * DB INs, including the DB root. When lruOnly=false, root IN 774 * eviction may not occur unless a cursor is used to pin the LN. 775 */ 776 env.evictMemory(); 777 778 c.close(); 779 } 780 781 TestUtils.validateNodeMemUsage(envImpl, true); 782 } 783 784 /** 785 * Open an environment and database. 786 */ openEnv(int floor, int maxMem)787 private void openEnv(int floor, 788 int maxMem) 789 throws DatabaseException { 790 791 EnvironmentConfig envConfig = 792 getEnvConfig(floor, maxMem, false /*readonly*/); 793 openEnv(envConfig); 794 } 795 796 /** 797 * Open an environment and database. 798 */ getEnvConfig(int floor, int maxMem, boolean readOnly)799 private EnvironmentConfig getEnvConfig(int floor, 800 int maxMem, 801 boolean readOnly) { 802 /* Convert floor percentage into bytes. */ 803 long evictBytes = 0; 804 if (floor > 0) { 805 evictBytes = maxMem - ((maxMem * floor) / 100); 806 } 807 808 /* Make a non-txnal env w/no daemons and small nodes. */ 809 EnvironmentConfig envConfig = TestUtils.initEnvConfig(); 810 envConfig.setAllowCreate(!readOnly); 811 envConfig.setReadOnly(readOnly); 812 envConfig.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC)); 813 envConfig.setConfigParam(EnvironmentParams. 814 ENV_RUN_EVICTOR.getName(), "false"); 815 envConfig.setConfigParam(EnvironmentParams. 816 ENV_RUN_INCOMPRESSOR.getName(), "false"); 817 envConfig.setConfigParam(EnvironmentParams. 818 ENV_RUN_CLEANER.getName(), "false"); 819 envConfig.setConfigParam(EnvironmentParams. 820 ENV_RUN_CHECKPOINTER.getName(), "false"); 821 if (evictBytes > 0) { 822 envConfig.setConfigParam(EnvironmentParams. 823 EVICTOR_EVICT_BYTES.getName(), 824 (new Long(evictBytes)).toString()); 825 } 826 envConfig.setConfigParam(EnvironmentParams. 827 MAX_MEMORY.getName(), 828 new Integer(maxMem).toString()); 829 /* Don't track detail with a tiny cache size. */ 830 envConfig.setConfigParam 831 (EnvironmentParams.CLEANER_TRACK_DETAIL.getName(), "false"); 832 envConfig.setConfigParam(EnvironmentParams.LOG_MEM_SIZE.getName(), 833 EnvironmentParams.LOG_MEM_SIZE_MIN_STRING); 834 envConfig.setConfigParam(EnvironmentParams.NUM_LOG_BUFFERS.getName(), 835 "2"); 836 /* Enable DB (MapLN) eviction for eviction tests. */ 837 envConfig.setConfigParam(EnvironmentParams. 838 ENV_DB_EVICTION.getName(), "true"); 839 840 /* 841 * Disable critical eviction, we want to test under controlled 842 * circumstances. 843 */ 844 envConfig.setConfigParam(EnvironmentParams. 845 EVICTOR_CRITICAL_PERCENTAGE.getName(), 846 "1000"); 847 848 /* Make small nodes */ 849 envConfig.setConfigParam(EnvironmentParams. 850 NODE_MAX.getName(), "4"); 851 envConfig.setConfigParam(EnvironmentParams. 852 NODE_MAX_DUPTREE.getName(), "4"); 853 854 return envConfig; 855 } 856 openEnv(EnvironmentConfig envConfig)857 private void openEnv(EnvironmentConfig envConfig) 858 throws DatabaseException { 859 860 env = new Environment(envHome, envConfig); 861 boolean readOnly = envConfig.getReadOnly(); 862 863 /* Open database. */ 864 DatabaseConfig dbConfig = new DatabaseConfig(); 865 dbConfig.setAllowCreate(!readOnly); 866 dbConfig.setReadOnly(readOnly); 867 dbConfig.setSortedDuplicates(true); 868 db = env.openDatabase(null, "foo", dbConfig); 869 } 870 closeEnv()871 private void closeEnv() 872 throws DatabaseException { 873 874 if (db != null) { 875 db.close(); 876 db = null; 877 } 878 if (env != null) { 879 env.close(); 880 env = null; 881 } 882 } 883 insertData(int nKeys)884 private void insertData(int nKeys) 885 throws DatabaseException { 886 887 DatabaseEntry key = new DatabaseEntry(); 888 DatabaseEntry data = new DatabaseEntry(); 889 for (int i = 0; i < nKeys; i++) { 890 891 IntegerBinding.intToEntry(i, key); 892 893 if ((i % 5) == 0) { 894 for (int j = 10; j < (NUM_DUPS + 10); j++) { 895 IntegerBinding.intToEntry(j, data); 896 db.put(null, key, data); 897 } 898 } else { 899 IntegerBinding.intToEntry(i+1, data); 900 db.put(null, key, data); 901 } 902 } 903 } 904 putLargeData(int nKeys, int dataSize)905 private void putLargeData(int nKeys, int dataSize) 906 throws DatabaseException { 907 908 DatabaseEntry key = new DatabaseEntry(); 909 DatabaseEntry data = new DatabaseEntry(new byte[dataSize]); 910 for (int i = 0; i < nKeys; i++) { 911 IntegerBinding.intToEntry(i, key); 912 db.put(null, key, data); 913 } 914 } 915 verifyData(int nKeys)916 private void verifyData(int nKeys) 917 throws DatabaseException { 918 919 /* Full scan of data, make sure we can bring everything back in. */ 920 Cursor cursor = db.openCursor(null, null); 921 DatabaseEntry data = new DatabaseEntry(); 922 DatabaseEntry key = new DatabaseEntry(); 923 924 for (int i = 0; i < nKeys; i++) { 925 if ((i % 5) ==0) { 926 for (int j = 10; j < (NUM_DUPS + 10); j++) { 927 assertEquals(OperationStatus.SUCCESS, 928 cursor.getNext(key, data, LockMode.DEFAULT)); 929 assertEquals(i, IntegerBinding.entryToInt(key)); 930 assertEquals(j, IntegerBinding.entryToInt(data)); 931 } 932 } else { 933 assertEquals(OperationStatus.SUCCESS, 934 cursor.getNext(key, data, LockMode.DEFAULT)); 935 assertEquals(i, IntegerBinding.entryToInt(key)); 936 assertEquals(i+1, IntegerBinding.entryToInt(data)); 937 } 938 } 939 940 assertEquals(OperationStatus.NOTFOUND, 941 cursor.getNext(key, data, LockMode.DEFAULT)); 942 cursor.close(); 943 } 944 evictAndCheck(boolean shouldEvict, int nKeys)945 private void evictAndCheck(boolean shouldEvict, int nKeys) 946 throws DatabaseException { 947 948 EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); 949 MemoryBudget mb = envImpl.getMemoryBudget(); 950 951 /* 952 * The following batches are run in a single evictMemory() call: 953 * 1st eviction will strip DBINs. 954 * 2nd will evict DBINs 955 * 3rd will evict DINs 956 * 4th will strip BINs 957 * 5th will evict BINs 958 * 6th will evict INs 959 * 7th will evict INs 960 */ 961 long preEvictMem = mb.getCacheMemoryUsage(); 962 TestUtils.validateNodeMemUsage(envImpl, true); 963 env.evictMemory(); 964 long postEvictMem = mb.getCacheMemoryUsage(); 965 966 TestUtils.validateNodeMemUsage(envImpl, true); 967 if (DEBUG) { 968 System.out.println("preEvict=" + preEvictMem + 969 " postEvict=" + postEvictMem); 970 } 971 972 if (shouldEvict) { 973 assertTrue("preEvict=" + preEvictMem + 974 " postEvict=" + postEvictMem + 975 " maxMem=" + mb.getMaxMemory(), 976 (preEvictMem > postEvictMem)); 977 } else { 978 assertTrue("preEvict=" + preEvictMem + 979 " postEvict=" + postEvictMem, 980 (preEvictMem == postEvictMem)); 981 } 982 983 verifyData(nKeys); 984 TestUtils.validateNodeMemUsage(envImpl, true); 985 } 986 } 987