1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.recovery;
9 
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertFalse;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertTrue;
15 
16 import java.io.File;
17 
18 import org.junit.After;
19 import org.junit.Test;
20 
21 import com.sleepycat.je.CacheMode;
22 import com.sleepycat.je.CheckpointConfig;
23 import com.sleepycat.je.DbInternal;
24 import com.sleepycat.je.Environment;
25 import com.sleepycat.je.EnvironmentConfig;
26 import com.sleepycat.je.config.EnvironmentParams;
27 import com.sleepycat.je.dbi.EnvironmentImpl;
28 import com.sleepycat.je.evictor.Evictor;
29 import com.sleepycat.je.evictor.Evictor.EvictionSource;
30 import com.sleepycat.je.log.FileManager;
31 import com.sleepycat.je.tree.BIN;
32 import com.sleepycat.je.tree.IN;
33 import com.sleepycat.je.util.TestUtils;
34 import com.sleepycat.je.utilint.DbLsn;
35 import com.sleepycat.je.utilint.TestHook;
36 import com.sleepycat.util.test.SharedTestUtils;
37 import com.sleepycat.util.test.TestBase;
38 
39 /**
40  * Tests coordination of eviction and checkpointing.
41  */
42 public class CkptEvictCoordTest extends TestBase {
43 
44     private static final boolean DEBUG = false;
45 
46     private File envHome;
47     private Environment env;
48     private EnvironmentImpl envImpl;
49 
CkptEvictCoordTest()50     public CkptEvictCoordTest() {
51         envHome = SharedTestUtils.getTestDir();
52     }
53 
54     @After
tearDown()55     public void tearDown() {
56 
57         try {
58             if (env != null) {
59                 env.close();
60             }
61         } catch (Throwable e) {
62             System.out.println("tearDown: " + e);
63         }
64 
65         env = null;
66         envImpl = null;
67         envHome = null;
68     }
69 
70     /**
71      * Opens the environment.
72      */
openEnv()73     private void openEnv() {
74 
75         EnvironmentConfig config = TestUtils.initEnvConfig();
76         config.setAllowCreate(true);
77 
78         /* Do not run the daemons. */
79         config.setConfigParam
80             (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
81         config.setConfigParam
82             (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
83         config.setConfigParam
84             (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false");
85         config.setConfigParam
86             (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false");
87         /* Set max batch files to one, for exact control over cleaning. */
88         config.setConfigParam
89             (EnvironmentParams.CLEANER_MAX_BATCH_FILES.getName(), "1");
90         /* Use a tiny log file size to write one IN per file. */
91         DbInternal.disableParameterValidation(config);
92         config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(),
93                               Integer.toString(64));
94 
95         /*
96          * Disable critical eviction, we want to test under controlled
97          * circumstances.
98          */
99         config.setConfigParam
100             (EnvironmentParams.EVICTOR_CRITICAL_PERCENTAGE.getName(), "1000");
101 
102         env = new Environment(envHome, config);
103         envImpl = DbInternal.getEnvironmentImpl(env);
104     }
105 
106     /**
107      * Closes the environment.
108      */
closeEnv()109     private void closeEnv() {
110 
111         if (env != null) {
112             env.close();
113             env = null;
114         }
115     }
116 
117     /**
118      * Verifies a fix for a LogFileNotFound issue that was introduced in JE 4.1
119      * by removing synchronization of eviction.  Eviction and the construction
120      * of the checkpointer dirty map were previously synchronized and could not
121      * execute concurrently.  Rather than synchronize them again, the fix is to
122      * make eviction log provisionally during construction of the dirty map,
123      * and add the dirty parent of the evicted IN to the dirty map.
124      *
125      * This test creates the scenario described here:
126      * https://sleepycat.oracle.com/trac/ticket/19346#comment:16
127      *
128      * [#19346]
129      */
130     @Test
testEvictionDuringDirtyMapCreation()131     public void testEvictionDuringDirtyMapCreation() {
132 
133         openEnv();
134 
135         /* Start with nothing dirty. */
136         env.sync();
137 
138         /*
139          * We use the FileSummaryDB because it just so happens that when we
140          * open a new environment, it has a single BIN and parent IN, and they
141          * are iterated by the INList in the order required to reproduce the
142          * problem: parent followed by child.  See the SR for details.
143          */
144         final long DB_ID = 2L; /* ID of FileSummaryDB is always 2. */
145 
146         /*
147          * Find parent IN and child BIN.  Check that IN precedes BIN when
148          * iterating the INList, which is necessary to reproduce the bug.
149          */
150         IN child = null;
151         IN parent = null;
152 
153         for (IN in : envImpl.getInMemoryINs()) {
154             if (in.getDatabase().getId().getId() == DB_ID) {
155                 if (in instanceof BIN) {
156                     assertNull("Expect only one BIN", child);
157                     child = in;
158                 } else {
159                     if (child != null) {
160                         System.out.println
161                             ("WARNING: Test cannot be performed because IN " +
162                              "parent does not precede child BIN");
163                         closeEnv();
164                         return;
165                     }
166                     assertNull("Expect only one IN", parent);
167                     parent = in;
168                 }
169             }
170         }
171         assertNotNull(child);
172         assertNotNull(parent);
173 
174         /* We use tiny log files so that each IN is in a different file. */
175         assertTrue(DbLsn.getFileNumber(child.getLastLoggedVersion()) !=
176                    DbLsn.getFileNumber(parent.getLastLoggedVersion()));
177 
178         if (DEBUG) {
179             System.out.println("child node=" + child.getNodeId() + " LSN=" +
180                                DbLsn.getNoFormatString
181                                (child.getLastLoggedVersion()) +
182                                " dirty=" + child.getDirty());
183             System.out.println("parent node=" + parent.getNodeId() + " LSN=" +
184                                DbLsn.getNoFormatString
185                                (parent.getLastLoggedVersion()) +
186                                " dirty=" + parent.getDirty());
187         }
188 
189         /*
190          * Clean the log file containing the child BIN.  Because we set
191          * CLEANER_MAX_BATCH_FILES to 1, this will clean only the single file
192          * that we inject below.
193          */
194         final long fileNum = DbLsn.getFileNumber(child.getLastLoggedVersion());
195         envImpl.getCleaner().getFileSelector().injectFileForCleaning(fileNum);
196 
197         final long filesCleaned = envImpl.getCleaner().doClean
198             (false /*cleanMultipleFiles*/, false /*forceCleaning*/);
199         assertEquals(1, filesCleaned);
200 
201         /* Parent must not be dirty.  Child must be dirty after cleaning. */
202         assertFalse(parent.getDirty());
203         assertTrue(child.getDirty());
204 
205         final IN useChild = child;
206         final IN useParent = parent;
207 
208         /* Hook called after examining each IN during dirty map creation. */
209         class DirtyMapHook implements TestHook<IN> {
210             boolean sawChild;
211             boolean sawParent;
212 
213             public void doHook(IN in) {
214 
215                 /*
216                  * The parent IN is iterated first, before the child BIN.  It
217                  * is not dirty, so it will not be added to the dirty map.  We
218                  * evict the child BIN at this time, so that the child will not
219                  * be added to the dirty map.
220                  *
221                  * The eviction creates the condition for the bug described in
222                  * the SR, which is that the child is logged in the checkpoint
223                  * interval but the parent is not, and the child is logged
224                  * non-provisionally.  With the bug fix, the child is logged
225                  * provisionally by the evictor and the parent is added to the
226                  * dirty map at that time, so that it will be logged by the
227                  * checkpoint.
228                  *
229                  * Ideally, to simulate real world conditions, this test should
230                  * do the eviction in a separate thread.  However, because
231                  * there was no synchronization between checkpointer and
232                  * evictor, the effect of doing it in the same thread is the
233                  * same.  Even with the bug fix, there is still no
234                  * synchronization between checkpointer and evictor at the time
235                  * the hook is called.
236                  */
237                 if (in == useParent) {
238                     assertFalse(sawChild);
239                     assertFalse(sawParent);
240                     assertFalse(in.getDirty());
241                     sawParent = true;
242 
243                     Evictor evictor = envImpl.getEvictor();
244 
245                     /*
246                      * First eviction strips LNs. Second evicts IN, if the old
247                      * evictor is used; otherwise, it moves the node to the
248                      * dirty LRUSet. As a result, one more eviction is needed
249                      * with the new evictor.
250                      */
251                     useChild.latch(CacheMode.UNCHANGED);
252                     evictor.doEvictOneIN(useChild, EvictionSource.MANUAL);
253 
254                     useChild.latch();
255                     evictor.doEvictOneIN(useChild, EvictionSource.MANUAL);
256 
257                     assertTrue(useChild.getInListResident());
258                     useChild.latch();
259                     evictor.doEvictOneIN(useChild, EvictionSource.MANUAL);
260 
261                     assertFalse(useChild.getInListResident());
262                 }
263 
264                 /*
265                  * We shouldn't see the child BIN because it was evicted, but
266                  * if we do see it then it should not be dirty.
267                  */
268                 if (in == useChild) {
269                     assertFalse(sawChild);
270                     assertTrue(sawParent);
271                     sawChild = true;
272                     assertFalse(in.getDirty());
273                 }
274             }
275 
276             /* Unused methods. */
277             public void doHook() {
278                 throw new UnsupportedOperationException();
279             }
280             public IN getHookValue() {
281                 throw new UnsupportedOperationException();
282             }
283             public void doIOHook() {
284                 throw new UnsupportedOperationException();
285             }
286             public void hookSetup() {
287                 throw new UnsupportedOperationException();
288             }
289         };
290 
291         /*
292          * Perform checkpoint and perform eviction during construction of the
293          * dirty map, using the hook above.
294          */
295         final DirtyMapHook hook = new DirtyMapHook();
296         Checkpointer.examineINForCheckpointHook = hook;
297         try {
298             env.checkpoint(new CheckpointConfig().setForce(true));
299         } finally {
300             Checkpointer.examineINForCheckpointHook = null;
301         }
302         assertTrue(hook.sawParent);
303 
304         /* Checkpoint should have deleted the file. */
305         final String fileName = envImpl.getFileManager().getFullFileName
306             (fileNum, FileManager.JE_SUFFIX);
307         assertFalse(fileName, new File(fileName).exists());
308 
309         /* Crash and recover. */
310         envImpl.abnormalClose();
311         envImpl = null;
312         /* Before the bug fix, this recovery caused LogFileNotFound. */
313         openEnv();
314         closeEnv();
315     }
316 }
317