1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.cleaner;
9 
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertTrue;
12 import static org.junit.Assert.fail;
13 
14 import java.io.File;
15 import java.nio.ByteBuffer;
16 import java.util.HashSet;
17 import java.util.Iterator;
18 import java.util.List;
19 import java.util.Set;
20 
21 import org.junit.After;
22 import org.junit.Test;
23 import org.junit.runner.RunWith;
24 import org.junit.runners.Parameterized;
25 import org.junit.runners.Parameterized.Parameters;
26 
27 import com.sleepycat.je.CacheMode;
28 import com.sleepycat.je.CheckpointConfig;
29 import com.sleepycat.je.Cursor;
30 import com.sleepycat.je.Database;
31 import com.sleepycat.je.DatabaseConfig;
32 import com.sleepycat.je.DatabaseEntry;
33 import com.sleepycat.je.DatabaseException;
34 import com.sleepycat.je.DbInternal;
35 import com.sleepycat.je.Environment;
36 import com.sleepycat.je.EnvironmentConfig;
37 import com.sleepycat.je.LockMode;
38 import com.sleepycat.je.OperationStatus;
39 import com.sleepycat.je.Transaction;
40 import com.sleepycat.je.config.EnvironmentParams;
41 import com.sleepycat.je.dbi.DatabaseId;
42 import com.sleepycat.je.dbi.DatabaseImpl;
43 import com.sleepycat.je.dbi.EnvironmentImpl;
44 import com.sleepycat.je.junit.JUnitThread;
45 import com.sleepycat.je.log.DumpFileReader;
46 import com.sleepycat.je.log.FileManager;
47 import com.sleepycat.je.log.LogEntryType;
48 import com.sleepycat.je.log.entry.INLogEntry;
49 import com.sleepycat.je.log.entry.LNLogEntry;
50 import com.sleepycat.je.log.entry.LogEntry;
51 import com.sleepycat.je.util.TestUtils;
52 import com.sleepycat.je.utilint.DbLsn;
53 import com.sleepycat.je.utilint.TestHook;
54 
55 /**
56  * Test cleaning and utilization counting for database truncate and remove.
57  */
58 @RunWith(Parameterized.class)
59 public class TruncateAndRemoveTest extends CleanerTestBase {
60 
61     private static final String DB_NAME1 = "foo";
62     private static final String DB_NAME2 = "bar";
63     private static final int RECORD_COUNT = 100;
64 
65     private static final CheckpointConfig FORCE_CHECKPOINT =
66         new CheckpointConfig();
67     static {
68         FORCE_CHECKPOINT.setForce(true);
69     }
70 
71     private static final boolean DEBUG = false;
72 
73     private EnvironmentImpl envImpl;
74     private Database db;
75     private DatabaseImpl dbImpl;
76     private JUnitThread junitThread;
77     private boolean fetchObsoleteSize;
78     private boolean truncateOrRemoveDone;
79     private boolean dbEviction;
80 
81     @Parameters
genParams()82     public static List<Object[]> genParams() {
83 
84         return getEnv(new boolean[] {false, true});
85     }
86 
TruncateAndRemoveTest(boolean envMultiDir)87     public TruncateAndRemoveTest (boolean envMultiDir) {
88         envMultiSubDir = envMultiDir;
89         customName = (envMultiSubDir) ? "multi-sub-dir" : null;
90     }
91 
92     @After
tearDown()93     public void tearDown()
94         throws Exception {
95 
96         if (junitThread != null) {
97             junitThread.shutdown();
98             junitThread = null;
99         }
100         super.tearDown();
101         db = null;
102         dbImpl = null;
103         envImpl = null;
104     }
105 
106     /**
107      * Opens the environment.
108      */
openEnv(boolean transactional)109     private void openEnv(boolean transactional)
110         throws DatabaseException {
111 
112         EnvironmentConfig config = TestUtils.initEnvConfig();
113         config.setTransactional(transactional);
114         config.setAllowCreate(true);
115         /* Do not run the daemons since they interfere with LN counting. */
116         config.setConfigParam
117             (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
118         config.setConfigParam
119             (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
120         config.setConfigParam
121             (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false");
122         config.setConfigParam
123             (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false");
124 
125         /* Use small nodes to test the post-txn scanning. */
126         config.setConfigParam
127             (EnvironmentParams.NODE_MAX.getName(), "10");
128         config.setConfigParam
129             (EnvironmentParams.NODE_MAX_DUPTREE.getName(), "10");
130         if (envMultiSubDir) {
131             config.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES,
132                                   DATA_DIRS + "");
133         }
134 
135         /* Use small files to ensure that there is cleaning. */
136         config.setConfigParam("je.cleaner.minUtilization", "80");
137         DbInternal.disableParameterValidation(config);
138         config.setConfigParam("je.log.fileMax", "4000");
139 
140         /* Obsolete LN size counting is optional per test. */
141         if (fetchObsoleteSize) {
142             config.setConfigParam
143                 (EnvironmentParams.CLEANER_FETCH_OBSOLETE_SIZE.getName(),
144                  "true");
145         }
146 
147         env = new Environment(envHome, config);
148         envImpl = DbInternal.getEnvironmentImpl(env);
149 
150         config = env.getConfig();
151         dbEviction = config.getConfigParam
152             (EnvironmentParams.ENV_DB_EVICTION.getName()).equals("true");
153     }
154 
155     /**
156      * Opens that database.
157      */
openDb(Transaction useTxn, String dbName)158     private void openDb(Transaction useTxn, String dbName)
159         throws DatabaseException {
160 
161         openDb(useTxn, dbName, null /*cacheMode*/);
162     }
163 
openDb(Transaction useTxn, String dbName, CacheMode cacheMode)164     private void openDb(Transaction useTxn, String dbName, CacheMode cacheMode)
165         throws DatabaseException {
166 
167         DatabaseConfig dbConfig = new DatabaseConfig();
168         EnvironmentConfig envConfig = env.getConfig();
169         dbConfig.setTransactional(envConfig.getTransactional());
170         dbConfig.setAllowCreate(true);
171         dbConfig.setCacheMode(cacheMode);
172         db = env.openDatabase(useTxn, dbName, dbConfig);
173         dbImpl = DbInternal.getDatabaseImpl(db);
174     }
175 
176     /**
177      * Closes the database.
178      */
closeDb()179     private void closeDb()
180         throws DatabaseException {
181 
182         if (db != null) {
183             db.close();
184             db = null;
185             dbImpl = null;
186         }
187     }
188 
189     /**
190      * Closes the environment and database.
191      */
closeEnv()192     private void closeEnv()
193         throws DatabaseException {
194 
195         closeDb();
196 
197         if (env != null) {
198             env.close();
199             env = null;
200             envImpl = null;
201         }
202     }
203 
204     @Test
testTruncate()205     public void testTruncate()
206         throws Exception {
207 
208         doTestTruncate(false /*simulateCrash*/);
209     }
210 
211     @Test
testTruncateRecover()212     public void testTruncateRecover()
213         throws Exception {
214 
215         doTestTruncate(true /*simulateCrash*/);
216     }
217 
218     /**
219      * Test that truncate generates the right number of obsolete LNs.
220      */
doTestTruncate(boolean simulateCrash)221     private void doTestTruncate(boolean simulateCrash)
222         throws Exception {
223 
224         openEnv(true);
225         openDb(null, DB_NAME1);
226         writeAndCountRecords(null, RECORD_COUNT);
227         DatabaseImpl saveDb = dbImpl;
228         DatabaseId saveId = dbImpl.getId();
229         closeDb();
230 
231         Transaction txn = env.beginTransaction(null, null);
232         truncate(txn, true);
233         ObsoleteCounts beforeCommit = getObsoleteCounts();
234         txn.commit();
235         truncateOrRemoveDone = true;
236 
237         /* Make sure use count is decremented when we commit. */
238         assertDbInUse(saveDb, false);
239         openDb(null, DB_NAME1);
240         saveDb = dbImpl;
241         closeDb();
242         assertDbInUse(saveDb, false);
243 
244         if (simulateCrash) {
245             envImpl.abnormalClose();
246             envImpl = null;
247             env = null;
248             openEnv(true);
249             /* After recovery, expect that the record LNs are obsolete. */
250             ObsoleteCounts afterCrash = getObsoleteCounts();
251             int obsolete = afterCrash.obsoleteLNs - beforeCommit.obsoleteLNs;
252             assertTrue("obsolete=" + obsolete + " expected=" + RECORD_COUNT,
253                        obsolete >= RECORD_COUNT);
254         } else {
255             verifyUtilization(beforeCommit,
256                               RECORD_COUNT + // LNs
257                               3,   // prev MapLN + deleted MapLN + prev NameLN
258                               15); // 1 root, 2 INs, 12 BINs
259         }
260 
261         closeEnv();
262         batchCleanAndVerify(saveId);
263     }
264 
265     /**
266      * Test that aborting truncate generates the right number of obsolete LNs.
267      */
268     @Test
testTruncateAbort()269     public void testTruncateAbort()
270         throws Exception {
271 
272         openEnv(true);
273         openDb(null, DB_NAME1);
274         writeAndCountRecords(null, RECORD_COUNT);
275         DatabaseImpl saveDb = dbImpl;
276         closeDb();
277 
278         Transaction txn = env.beginTransaction(null, null);
279         truncate(txn, true);
280         ObsoleteCounts beforeAbort = getObsoleteCounts();
281         txn.abort();
282 
283         /* Make sure use count is decremented when we abort. */
284         assertDbInUse(saveDb, false);
285         openDb(null, DB_NAME1);
286         saveDb = dbImpl;
287         closeDb();
288         assertDbInUse(saveDb, false);
289 
290         /*
291          * The obsolete count should include the records inserted after
292          * the truncate.
293          */
294         verifyUtilization(beforeAbort,
295                           /* 1 new nameLN, 2 copies of MapLN for new db */
296                            3,
297                            0);
298 
299         /* Reopen, db should be populated. */
300         openDb(null, DB_NAME1);
301         assertEquals(RECORD_COUNT, countRecords(null));
302         closeEnv();
303     }
304 
305     /**
306      * Test that aborting truncate generates the right number of obsolete LNs.
307      */
308     @Test
testTruncateRepopulateAbort()309     public void testTruncateRepopulateAbort()
310         throws Exception {
311 
312         openEnv(true);
313         openDb(null, DB_NAME1);
314         writeAndCountRecords(null, RECORD_COUNT);
315         closeDb();
316 
317         Transaction txn = env.beginTransaction(null, null);
318         truncate(txn, true);
319 
320         /* populate the database with some more records. */
321         openDb(txn, DB_NAME1);
322         writeAndCountRecords(txn, RECORD_COUNT/4);
323         DatabaseImpl saveDb = dbImpl;
324         DatabaseId saveId = dbImpl.getId();
325         closeDb();
326         ObsoleteCounts beforeAbort = getObsoleteCounts();
327         txn.abort();
328 
329         /*
330          * We set truncateOrRemoveDone to true (meaning that per-DB utilization
331          * will not be verified) even though the txn was aborted because the
332          * discarded new DatabaseImpl will not be counted yet includes INs and
333          * LNs from the operations above.
334          */
335         truncateOrRemoveDone = true;
336 
337         /* Make sure use count is decremented when we abort. */
338         assertDbInUse(saveDb, false);
339         openDb(null, DB_NAME1);
340         saveDb = dbImpl;
341         closeDb();
342         assertDbInUse(saveDb, false);
343 
344         /*
345          * The obsolete count should include the records inserted after
346          * the truncate.
347          */
348         verifyUtilization(beforeAbort,
349                           /* newly inserted LNs, 1 new nameLN,
350                            * 2 copies of MapLN for new db */
351                           (RECORD_COUNT/4) + 3,
352                           5);
353 
354         /* Reopen, db should be populated. */
355         openDb(null, DB_NAME1);
356         assertEquals(RECORD_COUNT, countRecords(null));
357 
358         closeEnv();
359         batchCleanAndVerify(saveId);
360     }
361 
362     @Test
testRemove()363     public void testRemove()
364         throws Exception {
365 
366         doTestRemove(false /*simulateCrash*/);
367     }
368 
369     @Test
testRemoveRecover()370     public void testRemoveRecover()
371         throws Exception {
372 
373         doTestRemove(true /*simulateCrash*/);
374     }
375 
376     /**
377      * Test that remove generates the right number of obsolete LNs.
378      */
doTestRemove(boolean simulateCrash)379     private void doTestRemove(boolean simulateCrash)
380         throws Exception {
381 
382         openEnv(true);
383         openDb(null, DB_NAME1);
384         writeAndCountRecords(null, RECORD_COUNT);
385         DatabaseImpl saveDb = dbImpl;
386         DatabaseId saveId = dbImpl.getId();
387         closeDb();
388 
389         Transaction txn = env.beginTransaction(null, null);
390         env.removeDatabase(txn, DB_NAME1);
391         ObsoleteCounts beforeCommit = getObsoleteCounts();
392         txn.commit();
393         truncateOrRemoveDone = true;
394 
395         /* Make sure use count is decremented when we commit. */
396         assertDbInUse(saveDb, false);
397 
398         if (simulateCrash) {
399             envImpl.abnormalClose();
400             envImpl = null;
401             env = null;
402             openEnv(true);
403             /* After recovery, expect that the record LNs are obsolete. */
404             ObsoleteCounts afterCrash = getObsoleteCounts();
405             int obsolete = afterCrash.obsoleteLNs - beforeCommit.obsoleteLNs;
406             assertTrue("obsolete=" + obsolete + " expected=" + RECORD_COUNT,
407                        obsolete >= RECORD_COUNT);
408         } else {
409             verifyUtilization(beforeCommit,
410                               /* LNs + old NameLN, old MapLN, delete MapLN */
411                               RECORD_COUNT + 3,
412                               15);
413         }
414 
415         openDb(null, DB_NAME1);
416         assertEquals(0, countRecords(null));
417 
418         closeEnv();
419         batchCleanAndVerify(saveId);
420     }
421 
422     /**
423      * Test that remove generates the right number of obsolete LNs.
424      */
425     @Test
testNonTxnalRemove()426     public void testNonTxnalRemove()
427         throws Exception {
428 
429         openEnv(false);
430         openDb(null, DB_NAME1);
431         writeAndCountRecords(null, RECORD_COUNT);
432         DatabaseImpl saveDb = dbImpl;
433         DatabaseId saveId = dbImpl.getId();
434         closeDb();
435         ObsoleteCounts beforeOperation = getObsoleteCounts();
436         env.removeDatabase(null, DB_NAME1);
437         truncateOrRemoveDone = true;
438 
439         /* Make sure use count is decremented. */
440         assertDbInUse(saveDb, false);
441 
442         verifyUtilization(beforeOperation,
443                           /* LNs + new NameLN, old NameLN, old MapLN, delete
444                              MapLN */
445                           RECORD_COUNT + 4,
446                           15);
447 
448         openDb(null, DB_NAME1);
449         assertEquals(0, countRecords(null));
450 
451         closeEnv();
452         batchCleanAndVerify(saveId);
453     }
454 
455     /**
456      * Test that aborting remove generates the right number of obsolete LNs.
457      */
458     @Test
testRemoveAbort()459     public void testRemoveAbort()
460         throws Exception {
461 
462         /* Create database, populate, remove, abort the remove. */
463         openEnv(true);
464         openDb(null, DB_NAME1);
465         writeAndCountRecords(null, RECORD_COUNT);
466         DatabaseImpl saveDb = dbImpl;
467         closeDb();
468         Transaction txn = env.beginTransaction(null, null);
469         env.removeDatabase(txn, DB_NAME1);
470         ObsoleteCounts beforeAbort = getObsoleteCounts();
471         txn.abort();
472 
473         /* Make sure use count is decremented when we abort. */
474         assertDbInUse(saveDb, false);
475 
476         verifyUtilization(beforeAbort, 0, 0);
477 
478         /* All records should be there. */
479         openDb(null, DB_NAME1);
480         assertEquals(RECORD_COUNT, countRecords(null));
481 
482         closeEnv();
483 
484         /*
485          * Batch clean and then check the record count again, just to make sure
486          * we don't lose any valid data.
487          */
488         openEnv(true);
489         while (env.cleanLog() > 0) {
490         }
491         CheckpointConfig force = new CheckpointConfig();
492         force.setForce(true);
493         env.checkpoint(force);
494         closeEnv();
495 
496         openEnv(true);
497         openDb(null, DB_NAME1);
498         assertEquals(RECORD_COUNT, countRecords(null));
499         closeEnv();
500     }
501 
502     /**
503      * The same as testRemoveNotResident but forces fetching of obsolets LNs
504      * in order to count their sizes accurately.
505      */
506     @Test
testRemoveNotResidentFetchObsoleteSize()507     public void testRemoveNotResidentFetchObsoleteSize()
508         throws Exception {
509 
510         fetchObsoleteSize = true;
511         testRemoveNotResident();
512     }
513 
514     /**
515      * Test that we can properly account for a non-resident database.
516      */
517     @Test
testRemoveNotResident()518     public void testRemoveNotResident()
519         throws Exception {
520 
521         /* Create a database, populate. */
522         openEnv(true);
523         /* Use EVICT_LN so that updates do not count obsolete size. */
524         openDb(null, DB_NAME1, CacheMode.EVICT_LN);
525         writeAndCountRecords(null, RECORD_COUNT);
526         /* Updates will not count obsolete size. */
527         writeAndCountRecords(null, RECORD_COUNT);
528         DatabaseId saveId = DbInternal.getDatabaseImpl(db).getId();
529         closeEnv();
530 
531         /*
532          * Open the environment and remove the database. The
533          * database is not resident at all.
534          */
535         openEnv(true);
536         Transaction txn = env.beginTransaction(null, null);
537         env.removeDatabase(txn, DB_NAME1);
538         ObsoleteCounts beforeCommit = getObsoleteCounts();
539         txn.commit();
540         truncateOrRemoveDone = true;
541 
542         verifyUtilization(beforeCommit,
543                           /* LNs + old NameLN, old MapLN, delete MapLN */
544                           RECORD_COUNT + 3,
545 
546                           /*
547                            * 15 INs for data tree, plus 2 for FileSummaryDB
548                            * split during tree walk.
549                            */
550                           DatabaseImpl.forceTreeWalkForTruncateAndRemove ?
551                           17 : 15,
552                           /* Records re-written + deleted + aborted LN. */
553                           RECORD_COUNT + 2,
554                           /* Records write twice. */
555                           RECORD_COUNT * 2,
556                           true /*expectAccurateObsoleteLNCount*/);
557 
558         /* check record count. */
559         openDb(null, DB_NAME1);
560         assertEquals(0, countRecords(null));
561 
562         closeEnv();
563         batchCleanAndVerify(saveId);
564     }
565 
566     /**
567      * The same as testRemovePartialResident but forces fetching of obsolets
568      * LNs in order to count their sizes accurately.
569      */
570     @Test
testRemovePartialResidentFetchObsoleteSize()571     public void testRemovePartialResidentFetchObsoleteSize()
572         throws Exception {
573 
574         fetchObsoleteSize = true;
575         testRemovePartialResident();
576     }
577 
578     /**
579      * Test that we can properly account for partially resident tree.
580      */
581     @Test
testRemovePartialResident()582     public void testRemovePartialResident()
583         throws Exception {
584 
585         /* Create a database, populate. */
586         openEnv(true);
587         /* Use EVICT_LN so that updates do not count obsolete size. */
588         openDb(null, DB_NAME1, CacheMode.EVICT_LN);
589         writeAndCountRecords(null, RECORD_COUNT);
590         /* Updates will not count obsolete size. */
591         writeAndCountRecords(null, RECORD_COUNT);
592         DatabaseId saveId = DbInternal.getDatabaseImpl(db).getId();
593         closeEnv();
594 
595         /*
596          * Open the environment and remove the database. Pull 1 BIN in.
597          */
598         openEnv(true);
599         openDb(null, DB_NAME1);
600         Cursor c = db.openCursor(null, null);
601         assertEquals(OperationStatus.SUCCESS,
602                      c.getFirst(new DatabaseEntry(), new DatabaseEntry(),
603                                 LockMode.DEFAULT));
604         c.close();
605         DatabaseImpl saveDb = dbImpl;
606         closeDb();
607 
608         Transaction txn = env.beginTransaction(null, null);
609         env.removeDatabase(txn, DB_NAME1);
610         ObsoleteCounts beforeCommit = getObsoleteCounts();
611         txn.commit();
612         truncateOrRemoveDone = true;
613 
614         /* Make sure use count is decremented when we commit. */
615         assertDbInUse(saveDb, false);
616 
617         verifyUtilization(beforeCommit,
618                           /* LNs + old NameLN, old MapLN, delete MapLN */
619                           RECORD_COUNT + 3,
620 
621                           /*
622                            * 15 INs for data tree, plus 2 for FileSummaryDB
623                            * split during tree walk.
624                            */
625                           DatabaseImpl.forceTreeWalkForTruncateAndRemove ?
626                           17 : 15,
627                           /* Records re-written + deleted + aborted LN. */
628                           RECORD_COUNT + 2,
629                           /* Records write twice. */
630                           RECORD_COUNT * 2,
631                           true /*expectAccurateObsoleteLNCount*/);
632 
633         /* check record count. */
634         openDb(null, DB_NAME1);
635         assertEquals(0, countRecords(null));
636 
637         closeEnv();
638         batchCleanAndVerify(saveId);
639     }
640 
641     /**
642      * Tests that a log file is not deleted by the cleaner when it contains
643      * entries in a database that is pending deletion.
644      */
645     @Test
testDBPendingDeletion()646     public void testDBPendingDeletion()
647         throws DatabaseException, InterruptedException {
648 
649         doDBPendingTest(RECORD_COUNT + 30, false /*deleteAll*/, 5);
650     }
651 
652     /**
653      * Like testDBPendingDeletion but creates a scenario where only a single
654      * log file is cleaned, and that log file contains only known obsolete
655      * log entries.  This reproduced a bug where we neglected to add pending
656      * deleted DBs to the cleaner's pending DB set if all entries in the log
657      * file were known obsoleted. [#13333]
658      */
659     @Test
testObsoleteLogFile()660     public void testObsoleteLogFile()
661         throws DatabaseException, InterruptedException {
662 
663         doDBPendingTest(70, true /*deleteAll*/, 1);
664     }
665 
doDBPendingTest(int recordCount, boolean deleteAll, int expectFilesCleaned)666     private void doDBPendingTest(int recordCount,
667                                  boolean deleteAll,
668                                  int expectFilesCleaned)
669         throws DatabaseException, InterruptedException {
670 
671         /* Create a database, populate, close. */
672         Set logFiles = new HashSet();
673         openEnv(true);
674         openDb(null, DB_NAME1);
675         writeAndMakeWaste(recordCount, logFiles, deleteAll);
676         int remainingRecordCount = deleteAll ? 0 : recordCount;
677         env.checkpoint(FORCE_CHECKPOINT);
678         ObsoleteCounts obsoleteCounts = getObsoleteCounts();
679         DatabaseImpl saveDb = dbImpl;
680         closeDb();
681         assertTrue(!saveDb.isDeleteFinished());
682         assertTrue(!saveDb.isDeleted());
683         assertDbInUse(saveDb, false);
684 
685         /* Make sure that we wrote a full file's worth of LNs. */
686         assertTrue(logFiles.size() >= 2);
687         assertTrue(logFilesExist(logFiles));
688 
689         /* Remove the database but do not commit yet. */
690         final Transaction txn = env.beginTransaction(null, null);
691         env.removeDatabase(txn, DB_NAME1);
692 
693         /*
694          * The obsolete count should be <= 1 (for the NameLN).
695          *
696          * The reaason for passing false for expectAccurateObsoleteLNCount is
697          * that the NameLN deletion is not committed.  It is not yet counted
698          * obsolete by live utilization counting, but will be counted obsolete
699          * by the utilization recalculation utility, which assumes that
700          * transactions will commit. [#22208]
701          */
702         obsoleteCounts = verifyUtilization
703             (obsoleteCounts, 1, 0, 0, 0,
704              false /*expectAccurateObsoleteLNCount*/);
705         truncateOrRemoveDone = true;
706 
707         junitThread = new JUnitThread("Committer") {
708             @Override
709             public void testBody() {
710                 try {
711                     txn.commit();
712                 } catch (Throwable e) {
713                     e.printStackTrace();
714                 }
715             }
716         };
717 
718         /*
719          * Set a hook to cause the commit to block.  The commit is done in a
720          * separate thread.  The commit will set the DB state to pendingDeleted
721          * and will then wait for the hook to return.
722          */
723         final Object lock = new Object();
724 
725         saveDb.setPendingDeletedHook(new TestHook() {
726             public void doHook() {
727                 synchronized (lock) {
728                     try {
729                         lock.notify();
730                         lock.wait();
731                     } catch (InterruptedException e) {
732                         e.printStackTrace();
733                         throw new RuntimeException(e.toString());
734                     }
735                 }
736             }
737             public Object getHookValue() {
738                 throw new UnsupportedOperationException();
739             }
740             public void doIOHook() {
741                 throw new UnsupportedOperationException();
742             }
743             public void hookSetup() {
744                 throw new UnsupportedOperationException();
745             }
746             public void doHook(Object obj) {
747                 throw new UnsupportedOperationException();
748             }
749         });
750 
751         /* Start the committer thread; expect the pending deleted state. */
752         synchronized (lock) {
753             junitThread.start();
754             lock.wait();
755         }
756         assertTrue(!saveDb.isDeleteFinished());
757         assertTrue(saveDb.isDeleted());
758         assertDbInUse(saveDb, true);
759 
760         /* Expect obsolete LNs: NameLN */
761         obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0);
762 
763         /* The DB deletion is pending; the log file should still exist. */
764         int filesCleaned = env.cleanLog();
765         assertEquals(expectFilesCleaned, filesCleaned);
766         assertTrue(filesCleaned > 0);
767         env.checkpoint(FORCE_CHECKPOINT);
768         env.checkpoint(FORCE_CHECKPOINT);
769         assertTrue(logFilesExist(logFiles));
770 
771         /*
772          * When the commiter thread finishes, the DB deletion will be
773          * complete and the DB state will change to deleted.
774          */
775         synchronized (lock) {
776             lock.notify();
777         }
778         try {
779             junitThread.finishTest();
780             junitThread = null;
781         } catch (Throwable e) {
782             e.printStackTrace();
783             fail(e.toString());
784         }
785         assertTrue(saveDb.isDeleteFinished());
786         assertTrue(saveDb.isDeleted());
787         assertDbInUse(saveDb, false);
788 
789         /* Expect obsolete LNs: recordCount + MapLN + FSLNs (apprx). */
790         verifyUtilization(obsoleteCounts, remainingRecordCount + 6, 0);
791 
792         /* The DB deletion is complete; the log file should be deleted. */
793         env.checkpoint(FORCE_CHECKPOINT);
794         env.checkpoint(FORCE_CHECKPOINT);
795         assertTrue(!logFilesExist(logFiles));
796     }
797 
798     /*
799      * The xxxForceTreeWalk tests set the DatabaseImpl
800      * forceTreeWalkForTruncateAndRemove field to true, which will force a walk
801      * of the tree to count utilization during truncate/remove, rather than
802      * using the per-database info.  This is used to test the "old technique"
803      * for counting utilization, which is now used only if the database was
804      * created prior to log version 6.
805      */
806     @Test
testTruncateForceTreeWalk()807     public void testTruncateForceTreeWalk()
808         throws Exception {
809 
810         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
811         try {
812             testTruncate();
813         } finally {
814             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
815         }
816     }
817 
818     @Test
testTruncateAbortForceTreeWalk()819     public void testTruncateAbortForceTreeWalk()
820         throws Exception {
821 
822         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
823         try {
824             testTruncateAbort();
825         } finally {
826             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
827         }
828     }
829 
830     @Test
testTruncateRepopulateAbortForceTreeWalk()831     public void testTruncateRepopulateAbortForceTreeWalk()
832         throws Exception {
833 
834         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
835         try {
836             testTruncateRepopulateAbort();
837         } finally {
838             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
839         }
840     }
841 
842     @Test
testRemoveForceTreeWalk()843     public void testRemoveForceTreeWalk()
844         throws Exception {
845 
846         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
847         try {
848             testRemove();
849         } finally {
850             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
851         }
852     }
853 
854     @Test
testNonTxnalRemoveForceTreeWalk()855     public void testNonTxnalRemoveForceTreeWalk()
856         throws Exception {
857 
858         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
859         try {
860             testNonTxnalRemove();
861         } finally {
862             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
863         }
864     }
865 
866     @Test
testRemoveAbortForceTreeWalk()867     public void testRemoveAbortForceTreeWalk()
868         throws Exception {
869 
870         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
871         try {
872             testRemoveAbort();
873         } finally {
874             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
875         }
876     }
877 
878     @Test
testRemoveNotResidentForceTreeWalk()879     public void testRemoveNotResidentForceTreeWalk()
880         throws Exception {
881 
882         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
883         try {
884             testRemoveNotResident();
885         } finally {
886             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
887         }
888     }
889 
890     @Test
testRemovePartialResidentForceTreeWalk()891     public void testRemovePartialResidentForceTreeWalk()
892         throws Exception {
893 
894         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
895         try {
896             testRemovePartialResident();
897         } finally {
898             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
899         }
900     }
901 
902     @Test
testDBPendingDeletionForceTreeWalk()903     public void testDBPendingDeletionForceTreeWalk()
904         throws Exception {
905 
906         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
907         try {
908             testDBPendingDeletion();
909         } finally {
910             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
911         }
912     }
913 
914     @Test
testObsoleteLogFileForceTreeWalk()915     public void testObsoleteLogFileForceTreeWalk()
916         throws Exception {
917 
918         DatabaseImpl.forceTreeWalkForTruncateAndRemove = true;
919         try {
920             testObsoleteLogFile();
921         } finally {
922             DatabaseImpl.forceTreeWalkForTruncateAndRemove = false;
923         }
924     }
925 
926     /**
927      * Tickles a bug that caused NPE during recovery during the sequence:
928      * delete record, trucate DB, crash (close without checkpoint), and
929      * recover. [#16515]
930      */
931     @Test
testDeleteTruncateRecover()932     public void testDeleteTruncateRecover()
933         throws DatabaseException {
934 
935         /* Delete a record. */
936         openEnv(true);
937         openDb(null, DB_NAME1);
938         writeAndCountRecords(null, 1);
939         closeDb();
940 
941         /* Truncate DB. */
942         Transaction txn = env.beginTransaction(null, null);
943         truncate(txn, false);
944         txn.commit();
945 
946         /* Close without checkpoint. */
947         envImpl.close(false /*doCheckpoint*/);
948         envImpl = null;
949         env = null;
950 
951         /* Recover -- the bug cause NPE here. */
952         openEnv(true);
953         closeEnv();
954     }
955 
writeAndCountRecords(Transaction txn, long count)956     private void writeAndCountRecords(Transaction txn, long count)
957         throws DatabaseException {
958 
959         for (int i = 1; i <= count; i += 1) {
960             DatabaseEntry entry = new DatabaseEntry(TestUtils.getTestArray(i));
961 
962             db.put(txn, entry, entry);
963         }
964 
965         /* Insert and delete some records, insert and abort some records. */
966         DatabaseEntry entry =
967             new DatabaseEntry(TestUtils.getTestArray((int)count+1));
968         db.put(txn, entry, entry);
969         db.delete(txn, entry);
970 
971         EnvironmentConfig envConfig = env.getConfig();
972         if (envConfig.getTransactional()) {
973             entry = new DatabaseEntry(TestUtils.getTestArray(0));
974             Transaction txn2 = env.beginTransaction(null, null);
975             db.put(txn2, entry, entry);
976             txn2.abort();
977             txn2 = null;
978         }
979 
980         assertEquals(count, countRecords(txn));
981     }
982 
983     /**
984      * Writes the specified number of records to db.  Check the number of
985      * records, and return the number of obsolete records.  Returns a set of
986      * the file numbers that are written to.
987      *
988      * Makes waste (obsolete records):  If doDelete=true, deletes records as
989      * they are added; otherwise does updates to produce obsolete records
990      * interleaved with non-obsolete records.
991      */
writeAndMakeWaste(long count, Set logFilesWritten, boolean doDelete)992     private void writeAndMakeWaste(long count,
993                                    Set logFilesWritten,
994                                    boolean doDelete)
995         throws DatabaseException {
996 
997         Transaction txn = env.beginTransaction(null, null);
998         Cursor cursor = db.openCursor(txn, null);
999         for (int i = 0; i < count; i += 1) {
1000             DatabaseEntry entry = new DatabaseEntry(TestUtils.getTestArray(i));
1001             cursor.put(entry, entry);
1002             /* Add log file written. */
1003             long file = CleanerTestUtils.getLogFile(cursor);
1004             logFilesWritten.add(new Long(file));
1005             /* Make waste. */
1006             if (!doDelete) {
1007                 cursor.put(entry, entry);
1008                 cursor.put(entry, entry);
1009             }
1010         }
1011         if (doDelete) {
1012             DatabaseEntry key = new DatabaseEntry();
1013             DatabaseEntry data = new DatabaseEntry();
1014             OperationStatus status;
1015             for (status = cursor.getFirst(key, data, null);
1016                  status == OperationStatus.SUCCESS;
1017                  status = cursor.getNext(key, data, null)) {
1018                 /* Make waste. */
1019                 cursor.delete();
1020                 /* Add log file written. */
1021                 long file = CleanerTestUtils.getLogFile(cursor);
1022                 logFilesWritten.add(new Long(file));
1023             }
1024         }
1025         cursor.close();
1026         txn.commit();
1027         assertEquals(doDelete ? 0 : count, countRecords(null));
1028     }
1029 
1030     /* Truncate database and check the count. */
truncate(Transaction useTxn, boolean getCount)1031     private void truncate(Transaction useTxn, boolean getCount)
1032         throws DatabaseException {
1033 
1034         long nTruncated = env.truncateDatabase(useTxn, DB_NAME1, getCount);
1035 
1036         if (getCount) {
1037             assertEquals(RECORD_COUNT, nTruncated);
1038         }
1039 
1040         assertEquals(0, countRecords(useTxn));
1041     }
1042 
1043     /**
1044      * Returns how many records are in the database.
1045      */
countRecords(Transaction useTxn)1046     private int countRecords(Transaction useTxn)
1047         throws DatabaseException {
1048 
1049         DatabaseEntry key = new DatabaseEntry();
1050         DatabaseEntry data = new DatabaseEntry();
1051         boolean opened = false;
1052         if (db == null) {
1053             openDb(useTxn, DB_NAME1);
1054             opened = true;
1055         }
1056         Cursor cursor = db.openCursor(useTxn, null);
1057         int count = 0;
1058         try {
1059             OperationStatus status = cursor.getFirst(key, data, null);
1060             while (status == OperationStatus.SUCCESS) {
1061                 count += 1;
1062                 status = cursor.getNext(key, data, null);
1063             }
1064         } finally {
1065             cursor.close();
1066         }
1067         if (opened) {
1068             closeDb();
1069         }
1070         return count;
1071     }
1072 
1073     /**
1074      * Return the total number of obsolete node counts according to the
1075      * UtilizationProfile and UtilizationTracker.
1076      */
getObsoleteCounts()1077     private ObsoleteCounts getObsoleteCounts() {
1078         FileSummary[] files = envImpl.getUtilizationProfile()
1079                .getFileSummaryMap(true)
1080                .values().toArray(new FileSummary[0]);
1081         int lnCount = 0;
1082         int inCount = 0;
1083         int lnSize = 0;
1084         int lnSizeCounted = 0;
1085         for (int i = 0; i < files.length; i += 1) {
1086             lnCount += files[i].obsoleteLNCount;
1087             inCount += files[i].obsoleteINCount;
1088             lnSize += files[i].obsoleteLNSize;
1089             lnSizeCounted += files[i].obsoleteLNSizeCounted;
1090         }
1091 
1092         return new ObsoleteCounts(lnCount, inCount, lnSize, lnSizeCounted);
1093     }
1094 
1095     private class ObsoleteCounts {
1096         int obsoleteLNs;
1097         int obsoleteINs;
1098         int obsoleteLNSize;
1099         int obsoleteLNSizeCounted;
1100 
ObsoleteCounts(int obsoleteLNs, int obsoleteINs, int obsoleteLNSize, int obsoleteLNSizeCounted)1101         ObsoleteCounts(int obsoleteLNs,
1102                        int obsoleteINs,
1103                        int obsoleteLNSize,
1104                        int obsoleteLNSizeCounted) {
1105             this.obsoleteLNs = obsoleteLNs;
1106             this.obsoleteINs = obsoleteINs;
1107             this.obsoleteLNSize = obsoleteLNSize;
1108             this.obsoleteLNSizeCounted = obsoleteLNSizeCounted;
1109         }
1110 
1111         @Override
toString()1112         public String toString() {
1113             return "lns=" + obsoleteLNs + " ins=" + obsoleteINs +
1114                    " lnSize=" + obsoleteLNSize +
1115                    " lnSizeCounted=" + obsoleteLNSizeCounted;
1116         }
1117     }
1118 
verifyUtilization(ObsoleteCounts prev, int expectedLNs, int expectedINs)1119     private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
1120                                              int expectedLNs,
1121                                              int expectedINs)
1122         throws DatabaseException {
1123 
1124         return verifyUtilization(prev, expectedLNs, expectedINs, 0, 0,
1125                                  true /*expectAccurateObsoleteLNCount*/);
1126     }
1127 
1128     /*
1129      * Check obsolete counts. If the expected IN count is zero, don't
1130      * check the obsolete IN count.  Always check the obsolete LN count.
1131      */
verifyUtilization(ObsoleteCounts prev, int expectedLNs, int expectedINs, int expectLNsSizeNotCounted, int minTotalLNsObsolete, boolean expectAccurateObsoleteLNCount)1132     private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
1133                                              int expectedLNs,
1134                                              int expectedINs,
1135                                              int expectLNsSizeNotCounted,
1136                                              int minTotalLNsObsolete,
1137                                          boolean expectAccurateObsoleteLNCount)
1138         throws DatabaseException {
1139 
1140         /*
1141          * If we are not forcing a tree walk or we have explicitly configured
1142          * fetchObsoleteSize, then the size of every LN should have been
1143          * counted.
1144          */
1145         boolean expectAccurateObsoleteLNSize =
1146             !DatabaseImpl.forceTreeWalkForTruncateAndRemove ||
1147             fetchObsoleteSize;
1148 
1149         /*
1150          * Unless we are forcing the tree walk and not not fetching to get
1151          * obsolete size, the obsolete size is always counted.
1152          */
1153         if (fetchObsoleteSize ||
1154             !DatabaseImpl.forceTreeWalkForTruncateAndRemove) {
1155             expectLNsSizeNotCounted = 0;
1156         }
1157 
1158         ObsoleteCounts now = getObsoleteCounts();
1159         String beforeAndAfter = "before: " + prev + " now: " + now;
1160 
1161         final int newObsolete = now.obsoleteLNs - prev.obsoleteLNs;
1162         assertEquals(beforeAndAfter, expectedLNs, newObsolete);
1163         if (expectAccurateObsoleteLNSize) {
1164             assertEquals(beforeAndAfter,
1165                          newObsolete + expectLNsSizeNotCounted,
1166                          now.obsoleteLNSizeCounted -
1167                          prev.obsoleteLNSizeCounted);
1168             final int expectMinSize = minTotalLNsObsolete * 6 /*average*/;
1169             assertTrue("expect min = " + expectMinSize +
1170                        " total size = " + now.obsoleteLNSize,
1171                        now.obsoleteLNSize > expectMinSize);
1172         }
1173 
1174         if (expectedINs > 0) {
1175             assertEquals(beforeAndAfter, expectedINs,
1176                          now.obsoleteINs - prev.obsoleteINs);
1177         }
1178 
1179         /*
1180          * We pass expectAccurateDbUtilization as false when
1181          * truncateOrRemoveDone, because the database utilization info for that
1182          * database is now gone.
1183          */
1184         VerifyUtils.verifyUtilization
1185             (envImpl,
1186              expectAccurateObsoleteLNCount,
1187              expectAccurateObsoleteLNSize,
1188              !truncateOrRemoveDone); // expectAccurateDbUtilization
1189 
1190         return now;
1191     }
1192 
1193     /**
1194      * Checks whether a given DB has a non-zero use count.  Does nothing if
1195      * je.dbEviction is not enabled, since reference counts are only maintained
1196      * if that config parameter is enabled.
1197      */
assertDbInUse(DatabaseImpl db, boolean inUse)1198     private void assertDbInUse(DatabaseImpl db, boolean inUse) {
1199         if (dbEviction) {
1200             assertEquals(inUse, db.isInUse());
1201         }
1202     }
1203 
1204     /**
1205      * Returns true if all files exist, or false if any file is deleted.
1206      */
logFilesExist(Set fileNumbers)1207     private boolean logFilesExist(Set fileNumbers) {
1208 
1209         Iterator iter = fileNumbers.iterator();
1210         while (iter.hasNext()) {
1211             long fileNum = ((Long) iter.next()).longValue();
1212             File file = new File(envImpl.getFileManager().getFullFileName
1213                                  (fileNum, FileManager.JE_SUFFIX));
1214             if (!file.exists()) {
1215                 return false;
1216             }
1217         }
1218         return true;
1219     }
1220 
1221     /*
1222      * Run batch cleaning and verify that there are no files with these
1223      * log entries.
1224      */
batchCleanAndVerify(DatabaseId dbId)1225     private void batchCleanAndVerify(DatabaseId dbId)
1226         throws Exception {
1227 
1228         /*
1229          * Open the environment, flip the log files to reduce mixing of new
1230          * records and old records and add more records to force the
1231          * utilization level of the removed records down.
1232          */
1233         openEnv(true);
1234         openDb(null, DB_NAME2);
1235         long lsn = envImpl.forceLogFileFlip();
1236         CheckpointConfig force = new CheckpointConfig();
1237         force.setForce(true);
1238         env.checkpoint(force);
1239 
1240         writeAndCountRecords(null, RECORD_COUNT * 3);
1241         env.checkpoint(force);
1242 
1243         closeDb();
1244 
1245         /* Check log files, there should be entries with this database. */
1246         CheckReader checker = new CheckReader(envImpl, dbId, true);
1247         while (checker.readNextEntry()) {
1248         }
1249 
1250         if (DEBUG) {
1251             System.out.println("entries for this db =" + checker.getCount());
1252         }
1253 
1254         assertTrue(checker.getCount() > 0);
1255 
1256         /* batch clean. */
1257         boolean anyCleaned = false;
1258         while (env.cleanLog() > 0) {
1259             anyCleaned = true;
1260         }
1261 
1262         assertTrue(anyCleaned);
1263 
1264         if (anyCleaned) {
1265             env.checkpoint(force);
1266         }
1267 
1268         /* Check log files, there should be no entries with this database. */
1269         checker = new CheckReader(envImpl, dbId, false);
1270         while (checker.readNextEntry()) {
1271         }
1272 
1273         closeEnv();
1274     }
1275 
1276     class CheckReader extends DumpFileReader{
1277 
1278         private final DatabaseId dbId;
1279         private final boolean expectEntries;
1280         private int count;
1281 
1282         /*
1283          * @param databaseId we're looking for log entries for this database.
1284          * @param expectEntries if false, there should be no log entries
1285          * with this database id. If true, the log should have entries
1286          * with this database id.
1287          */
CheckReader(EnvironmentImpl envImpl, DatabaseId dbId, boolean expectEntries)1288         CheckReader(EnvironmentImpl envImpl,
1289                     DatabaseId dbId,
1290                     boolean expectEntries)
1291             throws DatabaseException {
1292 
1293             super(envImpl, 1000, DbLsn.NULL_LSN, DbLsn.NULL_LSN,
1294                   DbLsn.NULL_LSN, null, null, false, false, true);
1295             this.dbId = dbId;
1296             this.expectEntries = expectEntries;
1297         }
1298 
1299         @Override
processEntry(ByteBuffer entryBuffer)1300         protected boolean processEntry(ByteBuffer entryBuffer)
1301             throws DatabaseException {
1302 
1303             /* Figure out what kind of log entry this is */
1304             byte type = currentEntryHeader.getType();
1305             LogEntryType lastEntryType = LogEntryType.findType(type);
1306             boolean isNode = lastEntryType.isNodeType();
1307 
1308             /* Read the entry. */
1309             LogEntry entry = lastEntryType.getSharedLogEntry();
1310             entry.readEntry(envImpl, currentEntryHeader, entryBuffer);
1311 
1312             long lsn = getLastLsn();
1313             if (isNode) {
1314                 boolean found = false;
1315                 if (entry instanceof INLogEntry) {
1316                     INLogEntry<?> inEntry = (INLogEntry<?>) entry;
1317                     found = dbId.equals(inEntry.getDbId());
1318                 } else {
1319                     LNLogEntry<?> lnEntry = (LNLogEntry<?>) entry;
1320                     found = dbId.equals(lnEntry.getDbId());
1321                 }
1322                 if (found) {
1323                     if (expectEntries) {
1324                         count++;
1325                     } else {
1326                         StringBuilder sb = new StringBuilder();
1327                         entry.dumpEntry(sb, false);
1328                         fail("lsn=" + DbLsn.getNoFormatString(lsn) +
1329                              " dbId = " + dbId +
1330                              " entry= " + sb.toString());
1331                     }
1332                 }
1333             }
1334 
1335             return true;
1336         }
1337 
1338         /* Num entries with this database id seen by reader. */
getCount()1339         int getCount() {
1340             return count;
1341         }
1342     }
1343 }
1344