1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.rep.vlsn;
9 
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertTrue;
12 
13 import java.io.File;
14 import java.util.ArrayList;
15 import java.util.Collections;
16 import java.util.Iterator;
17 import java.util.List;
18 import java.util.Set;
19 
20 import org.junit.Test;
21 
22 import com.sleepycat.je.DbInternal;
23 import com.sleepycat.je.Durability;
24 import com.sleepycat.je.Environment;
25 import com.sleepycat.je.EnvironmentConfig;
26 import com.sleepycat.je.dbi.EnvironmentImpl;
27 import com.sleepycat.je.log.LogEntryHeader;
28 import com.sleepycat.je.log.LogEntryType;
29 import com.sleepycat.je.log.LogItem;
30 import com.sleepycat.je.log.Provisional;
31 import com.sleepycat.je.log.ReplicationContext;
32 import com.sleepycat.je.log.entry.LogEntry;
33 import com.sleepycat.je.log.entry.SingleItemEntry;
34 import com.sleepycat.je.recovery.RecoveryInfo;
35 import com.sleepycat.je.rep.impl.node.NameIdPair;
36 import com.sleepycat.je.rep.vlsn.VLSNIndex.ForwardVLSNScanner;
37 import com.sleepycat.je.txn.RollbackStart;
38 import com.sleepycat.je.utilint.DbLsn;
39 import com.sleepycat.je.utilint.VLSN;
40 import com.sleepycat.util.test.SharedTestUtils;
41 import com.sleepycat.util.test.TestBase;
42 
43 public class MergeTest extends TestBase {
44 
45     private final String testMapDb = "TEST_MAP_DB";
46     private final boolean verbose = Boolean.getBoolean("verbose");
47     private final File envHome;
48     private final byte lnType =
49         LogEntryType.LOG_INS_LN_TRANSACTIONAL.getTypeNum();
50 
51     private Environment env;
52     private EnvironmentImpl envImpl;
53     private int bucketStride = 4;
54     private int bucketMaxMappings = 3;
55     private int recoveryStride = 3;
56     private int recoveryMaxMappings = 4;
57 
58 
MergeTest()59     public MergeTest() {
60         envHome = SharedTestUtils.getTestDir();
61     }
62 
makeEnvironment()63     private Environment makeEnvironment() {
64 
65         EnvironmentConfig envConfig = new EnvironmentConfig();
66         envConfig.setAllowCreate(true);
67         envConfig.setTransactional(false);
68         return new Environment(envHome, envConfig);
69     }
70 
71     /**
72      * Test the tricky business of recovering the VLSNIndex. See VLSNIndex(),
73      * and how the vlsnIndex is initialized from what's persistent on disk, and
74      * another tracker is filled with vlsn->lsn mappings gleaned from reading
75      * the log during recovery. The recovery tracker's contents are used to
76      * override what is on the on-disk tracker.
77      */
78     @Test
testMerge()79     public void testMerge()
80         throws Throwable {
81         env = makeEnvironment();
82         envImpl = DbInternal.getEnvironmentImpl(env);
83 
84         /*
85          * VLSN ranges for the test:
86          * start->initialSize are mapped before recovery
87          * secondStart ->recoverySize are mapped in the recovery tracker.
88          */
89         long start = 1;
90         long initialSize = 40;
91         long secondStart = start + initialSize - 20;
92         long recoverySize = 30;
93 
94         RecoveryTrackerGenerator generator =
95             new NoRollbackGenerator(secondStart, recoverySize);
96         try {
97             doMerge(generator, initialSize);
98         } finally {
99             env.close();
100         }
101     }
102 
103     @Test
testSingleRBMerge()104     public void testSingleRBMerge()
105         throws Throwable {
106         env = makeEnvironment();
107         envImpl = DbInternal.getEnvironmentImpl(env);
108 
109         long initialSize = 40;
110 
111         RecoveryTrackerGenerator generator = new RollbackGenerator
112             (new RollbackInfo(new VLSN(20), DbLsn.makeLsn(5, 20 * 10)));
113 
114         try {
115             doMerge(generator, initialSize);
116         } finally {
117             env.close();
118         }
119     }
120 
121     @Test
testMultiRBMerge()122     public void testMultiRBMerge()
123         throws Throwable {
124         env = makeEnvironment();
125         envImpl = DbInternal.getEnvironmentImpl(env);
126 
127         long initialSize = 50;
128 
129         RecoveryTrackerGenerator generator = new RollbackGenerator
130             (new RollbackInfo(new VLSN(30), DbLsn.makeLsn(5, 30 * 10)),
131              new TestInfo(31, DbLsn.makeLsn(6, (31 * 30))),
132              new TestInfo(32, DbLsn.makeLsn(6, (32 * 30))),
133              new TestInfo(33, DbLsn.makeLsn(6, (33 * 30))),
134              new TestInfo(34, DbLsn.makeLsn(6, (34 * 30))),
135              new TestInfo(35, DbLsn.makeLsn(6, (35 * 30))),
136              new TestInfo(36, DbLsn.makeLsn(6, (36 * 30))),
137              new TestInfo(37, DbLsn.makeLsn(6, (37 * 30))),
138              new RollbackInfo(new VLSN(33), DbLsn.makeLsn(6, (33 * 30))),
139              new TestInfo(34, DbLsn.makeLsn(7, (34 * 40))),
140              new TestInfo(35, DbLsn.makeLsn(7, (35 * 40))),
141              new TestInfo(36, DbLsn.makeLsn(7, (36 * 40))),
142              new TestInfo(37, DbLsn.makeLsn(7, (37 * 40))));
143 
144         try {
145             doMerge(generator, initialSize);
146         } finally {
147             env.close();
148         }
149     }
150 
151     @Test
testRBInRecoveryLogMerge()152     public void testRBInRecoveryLogMerge()
153         throws Throwable {
154         env = makeEnvironment();
155         envImpl = DbInternal.getEnvironmentImpl(env);
156 
157         long initialSize = 50;
158 
159         RecoveryTrackerGenerator generator = new RollbackGenerator
160             (new TestInfo(51, DbLsn.makeLsn(6, (51 * 30))),
161              new TestInfo(52, DbLsn.makeLsn(6, (52 * 30))),
162              new TestInfo(53, DbLsn.makeLsn(6, (53 * 30))),
163              new TestInfo(54, DbLsn.makeLsn(6, (54 * 30))),
164              new TestInfo(55, DbLsn.makeLsn(6, (55 * 30))),
165              new TestInfo(56, DbLsn.makeLsn(6, (56 * 30))),
166              new TestInfo(57, DbLsn.makeLsn(6, (57 * 30))),
167              new TestInfo(58, DbLsn.makeLsn(6, (58 * 30))),
168              new TestInfo(59, DbLsn.makeLsn(6, (59 * 30))),
169              new TestInfo(60, DbLsn.makeLsn(6, (60 * 30))),
170              new TestInfo(61, DbLsn.makeLsn(6, (61 * 30))),
171              new RollbackInfo(new VLSN(55), DbLsn.makeLsn(6, (55 * 30))),
172              new TestInfo(56, DbLsn.makeLsn(7, (56 * 40))),
173              new TestInfo(57, DbLsn.makeLsn(7, (57 * 40))));
174 
175         try {
176             doMerge(generator, initialSize);
177         } finally {
178             env.close();
179         }
180     }
181 
doMerge(RecoveryTrackerGenerator generator, long initialSize)182     private void doMerge(RecoveryTrackerGenerator generator,
183                          long initialSize)
184         throws Throwable {
185 
186         for (int flushPoint = 1; flushPoint <= initialSize; flushPoint++) {
187             if (verbose) {
188                 System.out.println("flush=" + flushPoint +
189                                    " initSize = " + initialSize);
190             }
191 
192             VLSNIndex vlsnIndex = new VLSNIndex(envImpl, testMapDb,
193                                                 new NameIdPair("node1", 1),
194                                                 bucketStride, bucketMaxMappings,
195                                                 10000, new RecoveryInfo());
196             try {
197 
198                 List<TestInfo> expected = new ArrayList<TestInfo>();
199 
200                 populate(flushPoint, vlsnIndex, initialSize, expected);
201                 vlsnIndex.merge(generator.makeRecoveryTracker(expected));
202 
203                 assertTrue(vlsnIndex.verify(verbose));
204                 checkMerge(vlsnIndex, expected);
205 
206             } catch (Throwable e) {
207                 e.printStackTrace();
208                 throw e;
209             } finally {
210                 vlsnIndex.close();
211                 env.removeDatabase(null, testMapDb);
212             }
213         }
214     }
215 
216     /**
217      * Fill up an initial VLSNIndex, flushing at different spots to create a
218      * different tracker/on-disk mix.
219      */
populate(int flushPoint, VLSNIndex vlsnIndex, long initialSize, List<TestInfo> expected)220     private void populate(int flushPoint,
221                           VLSNIndex vlsnIndex,
222                           long initialSize,
223                           List<TestInfo> expected) {
224 
225 
226         for (long i = 1; i <= initialSize; i++) {
227             TestInfo info = new TestInfo(i, DbLsn.makeLsn(5, i * 10));
228 
229             /* populate vlsn index */
230             vlsnIndex.put(makeLogItem(info));
231 
232             /* populate expected list */
233             expected.add(info);
234 
235             if (i == flushPoint) {
236                 vlsnIndex.flushToDatabase(Durability.COMMIT_NO_SYNC);
237             }
238         }
239     }
240 
makeLogItem(TestInfo info)241     private LogItem makeLogItem(TestInfo info) {
242         LogItem item = new LogItem();
243         item.header = info.header;
244         item.newLsn = info.lsn;
245         return item;
246     }
247 
checkMerge(VLSNIndex vlsnIndex, List<TestInfo> expected)248     private void checkMerge(VLSNIndex vlsnIndex, List<TestInfo> expected) {
249 
250         /* The new tracker should have the right range. */
251         VLSNRange range = vlsnIndex.getRange();
252         assertEquals(new VLSN(1), range.getFirst());
253         VLSN lastVLSN = expected.get(expected.size() - 1).vlsn;
254         assertEquals(lastVLSN, range.getLast());
255 
256         // TODO: test that the sync and commit fields in the tracker are
257         // correct.
258 
259         ForwardVLSNScanner scanner = new ForwardVLSNScanner(vlsnIndex);
260         long firstLsn = scanner.getStartingLsn(expected.get(0).vlsn);
261         assertEquals(DbLsn.getNoFormatString(expected.get(0).lsn) +
262                      " saw first VLSN " + DbLsn.getNoFormatString(firstLsn),
263                      expected.get(0).lsn, firstLsn);
264 
265         boolean vlsnForLastInRange = false;
266 
267         int validMappings = 0;
268         for (TestInfo info : expected) {
269             long lsn = scanner.getPreciseLsn(info.vlsn);
270             if (lsn != DbLsn.NULL_LSN) {
271                 if (verbose) {
272                     System.out.println(info);
273                 }
274 
275                 assertEquals(DbLsn.getNoFormatString(info.lsn), info.lsn, lsn);
276                 validMappings++;
277 
278                 if (info.vlsn.equals(lastVLSN)) {
279                     vlsnForLastInRange = true;
280                 }
281             }
282         }
283 
284         /* Should see a lsn value for the last VLSN in the range. */
285         assertTrue(vlsnForLastInRange);
286 
287         /* Some portion of the expected set should be mapped. */
288         assertTrue(validMappings > (expected.size()/bucketStride) - 1);
289     }
290 
291     interface RecoveryTrackerGenerator {
makeRecoveryTracker(List<TestInfo> expected)292         public VLSNRecoveryTracker makeRecoveryTracker(List<TestInfo> expected);
293     }
294 
295     private class NoRollbackGenerator implements RecoveryTrackerGenerator {
296 
297         private long secondStart;
298         private long recoverySize;
299 
NoRollbackGenerator(long secondStart, long recoverySize)300         NoRollbackGenerator(long secondStart,
301                             long recoverySize) {
302             this.secondStart = secondStart;
303             this.recoverySize = recoverySize;
304         }
305 
makeRecoveryTracker(List<TestInfo> expected)306         public VLSNRecoveryTracker makeRecoveryTracker
307             (List<TestInfo> expected) {
308 
309             VLSNRecoveryTracker recoveryTracker =
310                 new VLSNRecoveryTracker(envImpl,
311                                         recoveryStride,
312                                         recoveryMaxMappings,
313                                         100000);
314 
315             /* Truncate the expected mappings list. */
316             Iterator<TestInfo> iter = expected.iterator();
317             while (iter.hasNext()) {
318                 TestInfo ti = iter.next();
319                 if (ti.vlsn.getSequence() >= secondStart) {
320                     iter.remove();
321                 }
322             }
323 
324             for (long i = secondStart; i < secondStart + recoverySize; i ++) {
325                 TestInfo info = new TestInfo(i, DbLsn.makeLsn(6,i * 20));
326                 recoveryTracker.trackMapping(info.lsn,
327                                              info.header,
328                                              info.entry);
329                 expected.add(info);
330             }
331 
332             return recoveryTracker;
333         }
334     }
335 
336     private class RollbackGenerator implements RecoveryTrackerGenerator {
337         private final Object[] recoveryLog;
338 
RollbackGenerator(Object .... recoveryLog)339         RollbackGenerator(Object ... recoveryLog) {
340             this.recoveryLog = recoveryLog;
341         }
342 
343         public VLSNRecoveryTracker
makeRecoveryTracker(List<TestInfo> expected)344             makeRecoveryTracker(List<TestInfo> expected) {
345 
346             VLSNRecoveryTracker recoveryTracker =
347                 new VLSNRecoveryTracker(envImpl,
348                                         recoveryStride,
349                                         recoveryMaxMappings,
350                                         100000);
351 
352             for (Object info : recoveryLog) {
353                 if (info instanceof TestInfo) {
354                     TestInfo t = (TestInfo) info;
355                     recoveryTracker.trackMapping(t.lsn, t.header, t.entry);
356                     expected.add(t);
357                 } else if (info instanceof RollbackInfo) {
358                     RollbackInfo r = (RollbackInfo) info;
359 
360                     /* Register the pseudo rollback with the tracker. */
361                     recoveryTracker.trackMapping(0 /* lsn */,
362                                                  r.header, r.rollbackEntry);
363 
364                     /* Truncate the expected mappings list. */
365                     Iterator<TestInfo> iter = expected.iterator();
366                     while (iter.hasNext()) {
367                         TestInfo ti = iter.next();
368                         if (ti.vlsn.compareTo(r.matchpointVLSN) > 0) {
369                             iter.remove();
370                         }
371                     }
372                 }
373             }
374 
375             return recoveryTracker;
376         }
377     }
378 
379     private class TestInfo {
380         final long lsn;
381         final VLSN vlsn;
382         final LogEntryHeader header;
383         final LogEntry entry;
384 
TestInfo(long vlsnVal, long lsn, LogEntry entry)385         TestInfo(long vlsnVal, long lsn, LogEntry entry) {
386             this.lsn = lsn;
387             this.vlsn = new VLSN(vlsnVal);
388             this.header = new LogEntryHeader(entry.getLogType().getTypeNum(),
389                                              0, 0, vlsn);
390             this.entry = entry;
391         }
392 
TestInfo(long vlsnVal, long lsn)393         TestInfo(long vlsnVal, long lsn) {
394             this.lsn = lsn;
395             this.vlsn = new VLSN(vlsnVal);
396             this.header = new LogEntryHeader(lnType, 0, 0, vlsn);
397             this.entry = null;
398         }
399 
400         @Override
toString()401         public String toString() {
402             return "vlsn=" + vlsn + " lsn=" + DbLsn.getNoFormatString(lsn) +
403                 " entryType="  + header.getType();
404         }
405     }
406 
407     private class RollbackInfo {
408         final VLSN matchpointVLSN;
409         final LogEntryHeader header;
410         final LogEntry rollbackEntry;
411 
RollbackInfo(VLSN matchpointVLSN, long matchpointLsn)412         RollbackInfo(VLSN matchpointVLSN, long matchpointLsn) {
413 
414             this.matchpointVLSN = matchpointVLSN;
415             Set<Long> noActiveTxns = Collections.emptySet();
416             rollbackEntry =
417                 SingleItemEntry.create(LogEntryType.LOG_ROLLBACK_START,
418                                        new RollbackStart(matchpointVLSN,
419                                                          matchpointLsn,
420                                                          noActiveTxns));
421 
422              header = new LogEntryHeader(rollbackEntry, Provisional.NO,
423                                          ReplicationContext.NO_REPLICATE);
424         }
425     }
426 }
427