1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2013 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.dbi;
9 
10 import java.io.File;
11 
12 import static org.junit.Assert.assertEquals;
13 import static org.junit.Assert.assertFalse;
14 import static org.junit.Assert.assertNotNull;
15 import static org.junit.Assert.assertNotSame;
16 import static org.junit.Assert.assertSame;
17 import static org.junit.Assert.assertTrue;
18 
19 import com.sleepycat.je.CacheMode;
20 import com.sleepycat.je.CheckpointConfig;
21 import com.sleepycat.je.Cursor;
22 import com.sleepycat.je.Database;
23 import com.sleepycat.je.DatabaseConfig;
24 import com.sleepycat.je.DatabaseEntry;
25 import com.sleepycat.je.DbInternal;
26 import com.sleepycat.je.Durability;
27 import com.sleepycat.je.Environment;
28 import com.sleepycat.je.EnvironmentConfig;
29 import com.sleepycat.je.EnvironmentStats;
30 import com.sleepycat.je.OperationStatus;
31 import com.sleepycat.je.StatsConfig;
32 import com.sleepycat.je.Transaction;
33 import com.sleepycat.je.TransactionConfig;
34 import com.sleepycat.je.evictor.Evictor;
35 import com.sleepycat.je.tree.BIN;
36 import com.sleepycat.je.util.DualTestCase;
37 import com.sleepycat.je.util.TestUtils;
38 import com.sleepycat.util.test.SharedTestUtils;
39 import org.junit.Test;
40 
41 public class BINDeltaOperationTest extends DualTestCase {
42 
43     /*
44      * N_RECORDS is set to 110 and NODE_MAX_ENTRIES to 100. This results in
45      * a tree with 2 BIN, the first of which has 40 entries.
46      */
47     private static final int N_RECORDS = 110;
48 
49     private final File envHome;
50     private Environment env;
51     private Database db;
52 
BINDeltaOperationTest()53     public BINDeltaOperationTest() {
54         envHome = SharedTestUtils.getTestDir();
55     }
56 
open()57     private void open() {
58 
59         final EnvironmentConfig envConfig = TestUtils.initEnvConfig();
60         envConfig.setAllowCreate(true);
61         envConfig.setTransactional(true);
62         envConfig.setDurability(Durability.COMMIT_NO_SYNC);
63         envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_EVICTOR,
64             "false");
65         envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER,
66             "false");
67         envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CHECKPOINTER,
68             "false");
69         envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_IN_COMPRESSOR,
70             "false");
71         envConfig.setConfigParam(EnvironmentConfig.NODE_MAX_ENTRIES, "100");
72         env = create(envHome, envConfig);
73 
74         final DatabaseConfig dbConfig = new DatabaseConfig();
75         dbConfig.setAllowCreate(true);
76         dbConfig.setTransactional(true);
77         dbConfig.setCacheMode(CacheMode.EVICT_LN);
78         db = env.openDatabase(null, "testDB", dbConfig);
79     }
80 
close()81     private void close() {
82         db.close();
83         db = null;
84         env.close();
85         env = null;
86     }
87 
88     @Test
testEviction()89     public void testEviction() {
90 
91         open();
92 
93         /* Insert N_RECORDS records into 2 BINs */
94         writeData();
95         checkData(null);
96 
97         /* Flush BINs (checkpoint). */
98         env.checkpoint(new CheckpointConfig().setForce(true));
99         checkData(null);
100 
101         /*
102          * Update only enough records in the 1st BIN to make it a delta
103          * when it gets selected for eviction.
104          */
105         writeDeltaFraction();
106 
107         env.checkpoint(new CheckpointConfig().setForce(true));
108 
109 
110         BIN bin = getFirstBIN();
111         assertFalse(bin.isBINDelta(false));
112 
113         EnvironmentStats stats = env.getStats(StatsConfig.CLEAR);
114         final long initialBINs = stats.getNCachedBINs();
115         final long initialDeltas = stats.getNCachedBINDeltas();
116         assertEquals(0, initialDeltas);
117 
118         /*
119          * Test partial eviction to mutate full BIN to BIN delta.  Calling
120          * evict() once will mutate to a BIN delta.  (Note that if LNs were
121          * resident, the first call would only strip the LNs, but we're using
122          * CacheMode.EVICT_LN so no LNs are resident.)
123          */
124         evict(bin, false);
125         assertTrue(bin.isBINDelta(false));
126         assertTrue(bin.getInListResident());
127         stats = env.getStats(null);
128         assertEquals(initialBINs, stats.getNCachedBINs());
129         assertEquals(initialDeltas + 1, stats.getNCachedBINDeltas());
130         assertEquals(1, stats.getNBINsMutated());
131 
132         Transaction txn = env.beginTransaction(null, TransactionConfig.DEFAULT);
133         Cursor cursor1 = db.openCursor(txn, null);
134         Cursor cursor2 = db.openCursor(txn, null);
135 
136         /*
137          * Read 2 existing keys directly from the bin delta, using 2 different
138          * cursors; one doing an exact search and the other a range search.
139          */
140         searchExistingKey(cursor1, 6, true);
141         searchExistingKey(cursor2, 10, false);
142 
143         assertTrue(bin.isBINDelta(false));
144 
145         /*
146          * Update the record where cursor1 is positioned at and make sure the
147          * bin is still a delta.
148          */
149         updateCurrentRecord(cursor1, 20);
150 
151         assertTrue(bin.isBINDelta(false));
152 
153         /*
154          * Now, read all records (checkData). This will mutate the delta to a
155          * the full BIN, but only one fetch will be needed (nNotResident == 1).
156          */
157         checkData(txn);
158         assertTrue(!bin.isBINDelta(false));
159         assertSame(bin, getFirstBIN());
160         stats = env.getStats(StatsConfig.CLEAR);
161         assertEquals(initialBINs, stats.getNCachedBINs());
162         assertEquals(initialDeltas, stats.getNCachedBINDeltas());
163         assertEquals(1, stats.getNFullBINsMiss());
164         assertEquals(0, stats.getNBINDeltasFetchMiss());
165         assertEquals(1, stats.getNNotResident());
166 
167         /*
168          * Make sure that cursors 1 and 2 were adjusted correctly when the
169          * delta got mutated.
170          */
171         confirmCurrentKey(cursor1, 6);
172         confirmCurrentKey(cursor2, 10);
173 
174         cursor1.close();
175         cursor2.close();
176         txn.commit();
177 
178         /*
179          * Call evict() to mutate the BIN to a delta.
180          */
181         evict(bin, false);
182         assertTrue(bin.isBINDelta(false));
183         assertTrue(bin.getInListResident());
184         stats = env.getStats(null);
185         assertEquals(initialBINs, stats.getNCachedBINs());
186         assertEquals(initialDeltas + 1, stats.getNCachedBINDeltas());
187         assertEquals(1, stats.getNBINsMutated());
188 
189         /*
190          * Delete a record from the bin delta
191          */
192         txn = env.beginTransaction(null, TransactionConfig.DEFAULT);
193         cursor1 = db.openCursor(txn, null);
194 
195         searchExistingKey(cursor1, 6, true);
196         assertTrue(bin.isBINDelta(false));
197         assertEquals(OperationStatus.SUCCESS, cursor1.delete());
198 
199         cursor1.close();
200         txn.commit();
201 
202         assertTrue(bin.isBINDelta(false));
203 
204         /*
205          * Call evict(true) to evict the BIN completely (without explicitly
206          * forcing the eviction, the BIN will be put in the dirty LRU). Then
207          * reading the entries will require two fetches.
208          */
209         evict(bin, true);
210         assertFalse(bin.getInListResident());
211         stats = env.getStats(null);
212         assertEquals(initialBINs - 1, stats.getNCachedBINs());
213         assertEquals(initialDeltas, stats.getNCachedBINDeltas());
214 
215         BIN prevBin = bin;
216         bin = getFirstBIN();
217         assertNotSame(prevBin, bin);
218         assertFalse(bin.isBINDelta(false));
219         stats = env.getStats(StatsConfig.CLEAR);
220         assertEquals(initialBINs, stats.getNCachedBINs());
221         assertEquals(initialDeltas, stats.getNCachedBINDeltas());
222         assertEquals(1, stats.getNBINsFetchMiss());
223         assertEquals(1, stats.getNBINDeltasFetchMiss());
224         assertEquals(1, stats.getNFullBINsMiss());
225         assertEquals(2, stats.getNNotResident());
226 
227         /*
228          * Put back the record deleted above, so that checkData won't complain.
229          */
230         inserRecord(6, 10);
231 
232         checkData(null);
233         stats = env.getStats(StatsConfig.CLEAR);
234         assertEquals(0, stats.getNBINsFetchMiss());
235         assertEquals(0, stats.getNBINDeltasFetchMiss());
236         assertEquals(0, stats.getNNotResident());
237 
238         close();
239     }
240 
241     @Test
testDbCount()242     public void testDbCount() {
243 
244         open();
245 
246         writeData(0, 5000);
247         assertEquals(5000, db.count());
248         checkData(null, 5000);
249         close();
250 
251         open();
252         assertEquals(5000, db.count());
253         checkData(null, 5000);
254         close();
255     }
256 
writeData()257     private void writeData() {
258         writeData(0, N_RECORDS);
259     }
260 
writeDeltaFraction()261     private void writeDeltaFraction() {
262         /* Update records in slots 5 to 15 (inclusive) of the 1st BIN */
263         writeData(5, N_RECORDS / 10);
264     }
265 
writeData(int startRecord, int nRecords)266     private void writeData(int startRecord, int nRecords) {
267 
268         final DatabaseEntry key = new DatabaseEntry();
269         final DatabaseEntry data = new DatabaseEntry();
270 
271         for (int i = startRecord; i < nRecords + startRecord; i += 1) {
272 
273             key.setData(TestUtils.getTestArray(i));
274             data.setData(TestUtils.getTestArray(i));
275 
276             assertEquals(OperationStatus.SUCCESS, db.put(null, key, data));
277         }
278     }
279 
inserRecord(int keyVal, int dataVal)280     private void inserRecord(int keyVal, int dataVal) {
281 
282         final DatabaseEntry key = new DatabaseEntry();
283         final DatabaseEntry data = new DatabaseEntry();
284         key.setData(TestUtils.getTestArray(keyVal));
285         data.setData(TestUtils.getTestArray(dataVal));
286 
287         assertEquals(OperationStatus.SUCCESS, db.put(null, key, data));
288     }
289 
290 
checkData(final Transaction txn)291     private void checkData(final Transaction txn) {
292         checkData(txn, N_RECORDS);
293     }
294 
295     /**
296      * Reads all keys, but does not read data to avoid changing the
297      * nNotResident stat.
298      */
checkData(final Transaction txn, final int nRecords)299     private void checkData(final Transaction txn, final int nRecords) {
300 
301         final DatabaseEntry key = new DatabaseEntry();
302         final DatabaseEntry data = new DatabaseEntry();
303         data.setPartial(true);
304 
305         final Cursor cursor = db.openCursor(txn, null);
306 
307         for (int i = 0; i < nRecords; i += 1) {
308 
309             assertEquals(OperationStatus.SUCCESS,
310                          cursor.getNext(key, data, null));
311 
312             assertEquals(i, TestUtils.getTestVal(key.getData()));
313         }
314 
315         assertEquals(OperationStatus.NOTFOUND,
316                      cursor.getNext(key, data, null));
317 
318         cursor.close();
319     }
320 
searchExistingKey( final Cursor cursor, final int keyVal, boolean exactSearch)321     private void searchExistingKey(
322         final Cursor cursor,
323         final int keyVal,
324         boolean exactSearch) {
325 
326         final DatabaseEntry key = new DatabaseEntry();
327         final DatabaseEntry data = new DatabaseEntry();
328 
329         key.setData(TestUtils.getTestArray(keyVal));
330 
331         data.setPartial(true);
332 
333         if (exactSearch) {
334             assertEquals(OperationStatus.SUCCESS,
335                          cursor.getSearchKey(key, data, null));
336         } else {
337             assertEquals(OperationStatus.SUCCESS,
338                          cursor.getSearchKeyRange(key, data, null));
339         }
340 
341         assertEquals(keyVal, TestUtils.getTestVal(key.getData()));
342     }
343 
confirmCurrentKey(final Cursor cursor, int keyVal)344     private Cursor confirmCurrentKey(final Cursor cursor, int keyVal) {
345 
346         final DatabaseEntry key = new DatabaseEntry();
347         final DatabaseEntry data = new DatabaseEntry();
348         data.setPartial(true);
349 
350         assertEquals(OperationStatus.SUCCESS,
351                      cursor.getCurrent(key, data, null));
352 
353         assertEquals(keyVal, TestUtils.getTestVal(key.getData()));
354 
355         return cursor;
356     }
357 
updateCurrentRecord(final Cursor cursor, int dataVal)358     private void updateCurrentRecord(final Cursor cursor, int dataVal) {
359 
360         final DatabaseEntry data = new DatabaseEntry();
361         data.setData(TestUtils.getTestArray(dataVal));
362 
363         assertEquals(OperationStatus.SUCCESS, cursor.putCurrent(data));
364 
365         final DatabaseEntry key = new DatabaseEntry();
366         data.setData(null);
367 
368         assertEquals(OperationStatus.SUCCESS,
369                      cursor.getCurrent(key, data, null));
370 
371         assertEquals(dataVal, TestUtils.getTestVal(data.getData()));
372     }
373 
374     /**
375      * Reads first key and returns its BIN. Does not read data to avoid
376      * changing the nNotResident stat.
377      */
getFirstBIN()378     private BIN getFirstBIN() {
379 
380         final DatabaseEntry key = new DatabaseEntry();
381         final DatabaseEntry data = new DatabaseEntry();
382         data.setPartial(true);
383 
384         final Cursor cursor = db.openCursor(null, null);
385 
386         assertEquals(OperationStatus.SUCCESS,
387                      cursor.getFirst(key, data, null));
388 
389         final BIN bin = DbInternal.getCursorImpl(cursor).getBIN();
390         cursor.close();
391         assertNotNull(bin);
392         return bin;
393     }
394 
395     /**
396      * Simulated eviction of the BIN, if it were selected.  This may only do
397      * partial eviction, if LNs are present or it can be mutated to a delta.
398      * We expect that some memory will be reclaimed.
399      */
evict(BIN bin, boolean force)400     private void evict(BIN bin, boolean force) {
401 
402         EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
403         Evictor evictor = envImpl.getEvictor();
404 
405         final long memBefore = TestUtils.validateNodeMemUsage(envImpl, true);
406 
407         bin.latch(CacheMode.UNCHANGED);
408 
409         if (force) {
410             evictor.doEvictOneIN(bin, Evictor.EvictionSource.CACHEMODE);
411         } else {
412             evictor.doEvictOneIN(bin, Evictor.EvictionSource.MANUAL);
413         }
414 
415         final long memAfter = TestUtils.validateNodeMemUsage(envImpl, true);
416 
417         assertTrue(memAfter < memBefore);
418     }
419 }
420