1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.log;
9 
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertTrue;
12 import static org.junit.Assert.fail;
13 
14 import java.io.File;
15 import java.nio.ByteBuffer;
16 
17 import org.junit.After;
18 import org.junit.Test;
19 
20 import com.sleepycat.bind.tuple.IntegerBinding;
21 import com.sleepycat.je.CheckpointConfig;
22 import com.sleepycat.je.Database;
23 import com.sleepycat.je.DatabaseConfig;
24 import com.sleepycat.je.DatabaseEntry;
25 import com.sleepycat.je.DatabaseException;
26 import com.sleepycat.je.DbInternal;
27 import com.sleepycat.je.Environment;
28 import com.sleepycat.je.EnvironmentConfig;
29 import com.sleepycat.je.EnvironmentMutableConfig;
30 import com.sleepycat.je.EnvironmentStats;
31 import com.sleepycat.je.LockMode;
32 import com.sleepycat.je.OperationFailureException;
33 import com.sleepycat.je.OperationStatus;
34 import com.sleepycat.je.RunRecoveryException;
35 import com.sleepycat.je.Transaction;
36 import com.sleepycat.je.cleaner.UtilizationProfile;
37 import com.sleepycat.je.config.EnvironmentParams;
38 import com.sleepycat.je.dbi.EnvironmentImpl;
39 import com.sleepycat.je.util.TestUtils;
40 import com.sleepycat.je.utilint.DbLsn;
41 import com.sleepycat.util.test.SharedTestUtils;
42 import com.sleepycat.util.test.TestBase;
43 import com.sleepycat.utilint.StringUtils;
44 
45 public class IOExceptionTest extends TestBase {
46 
47     private Environment env;
48     private Database db;
49     private final File envHome;
50 
IOExceptionTest()51     public IOExceptionTest() {
52         envHome = SharedTestUtils.getTestDir();
53     }
54 
55     @After
tearDown()56     public void tearDown()
57         throws DatabaseException {
58 
59         FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
60         if (db != null) {
61             db.close();
62         }
63 
64         if (env != null) {
65             env.close();
66         }
67     }
68 
69     @Test
testRunRecoveryExceptionOnWrite()70     public void testRunRecoveryExceptionOnWrite() {
71         try {
72             createDatabase(200000, 0, false);
73 
74             final int N_RECS = 25;
75 
76             CheckpointConfig chkConf = new CheckpointConfig();
77             chkConf.setForce(true);
78             Transaction txn = env.beginTransaction(null, null);
79             int keyInt = 0;
80             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
81             for (int i = 0; i < N_RECS; i++) {
82                 String keyStr = Integer.toString(keyInt);
83                 DatabaseEntry key =
84                     new DatabaseEntry(StringUtils.toUTF8(keyStr));
85                 DatabaseEntry data =
86                     new DatabaseEntry(StringUtils.toUTF8(("d" + keyStr)));
87                 if (i == (N_RECS - 1)) {
88                     FileManager.THROW_RRE_FOR_UNIT_TESTS = true;
89                     FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
90                 }
91                 try {
92                     assertTrue(db.put(txn, key, data) ==
93                                OperationStatus.SUCCESS);
94                 } catch (DatabaseException DE) {
95                     fail("unexpected DatabaseException");
96                     break;
97                 }
98             }
99 
100             try {
101                 txn.commit();
102                 fail("expected DatabaseException");
103             } catch (RunRecoveryException DE) {
104             }
105             forceCloseEnvOnly();
106 
107             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
108             FileManager.THROW_RRE_FOR_UNIT_TESTS = false;
109             db = null;
110             env = null;
111         } catch (Throwable E) {
112             E.printStackTrace();
113         }
114     }
115 
116     @Test
testIOExceptionNoRecovery()117     public void testIOExceptionNoRecovery()
118         throws Throwable {
119 
120         doIOExceptionTest(false);
121     }
122 
123     @Test
testIOExceptionWithRecovery()124     public void testIOExceptionWithRecovery()
125         throws Throwable {
126 
127         doIOExceptionTest(true);
128     }
129 
130     @Test
testEviction()131     public void testEviction() {
132         try {
133             createDatabase(200000, 0, true);
134 
135             final int N_RECS = 25;
136 
137             CheckpointConfig chkConf = new CheckpointConfig();
138             chkConf.setForce(true);
139             Transaction txn = env.beginTransaction(null, null);
140             int keyInt = 0;
141             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
142             for (int i = 0; i < N_RECS; i++) {
143                 String keyStr = Integer.toString(keyInt);
144                 DatabaseEntry key =
145                     new DatabaseEntry(StringUtils.toUTF8(keyStr));
146                 DatabaseEntry data =
147                     new DatabaseEntry(StringUtils.toUTF8(("d" + keyStr)));
148                 try {
149                     assertTrue(db.put(txn, key, data) ==
150                                OperationStatus.SUCCESS);
151                 } catch (DatabaseException DE) {
152                     fail("unexpected DatabaseException");
153                     break;
154                 }
155             }
156 
157             try {
158                 env.checkpoint(chkConf);
159                 fail("expected DatabaseException");
160             } catch (DatabaseException DE) {
161             }
162 
163             EnvironmentStats stats = env.getStats(null);
164             assertTrue((stats.getNFullINFlush() +
165                         stats.getNFullBINFlush()) > 0);
166 
167             /* Read back the data and make sure it all looks ok. */
168             for (int i = 0; i < N_RECS; i++) {
169                 String keyStr = Integer.toString(keyInt);
170                 DatabaseEntry key =
171                     new DatabaseEntry(StringUtils.toUTF8(keyStr));
172                 DatabaseEntry data = new DatabaseEntry();
173                 try {
174                     assertTrue(db.get(txn, key, data, null) ==
175                                OperationStatus.SUCCESS);
176                     assertEquals(StringUtils.fromUTF8(data.getData()),
177                                  "d" + keyStr);
178                 } catch (DatabaseException DE) {
179                     fail("unexpected DatabaseException");
180                     break;
181                 }
182             }
183 
184             /*
185              * Now we have some IN's in the log buffer and there have been
186              * IOExceptions that will later force rewriting that buffer.
187              */
188             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
189             try {
190                 txn.commit();
191             } catch (DatabaseException DE) {
192                 fail("unexpected DatabaseException");
193             }
194         } catch (Exception E) {
195             E.printStackTrace();
196         }
197     }
198 
199     /*
200      * Test for SR 13898.  Write out some records with
201      * IO_EXCEPTION_TESTING_ON_WRITE true thereby forcing some commits to be
202      * rewritten as aborts.  Ensure that the checksums are correct on those
203      * rewritten records by reading them back with a file reader.
204      */
205     @Test
testIOExceptionReadBack()206     public void testIOExceptionReadBack()
207         throws Exception {
208 
209         createDatabase(100000, 1000, true);
210 
211         /*
212          * Turn off daemons so we can check the size of the log
213          * deterministically.
214          */
215         EnvironmentMutableConfig newConfig = new EnvironmentMutableConfig();
216         newConfig.setConfigParam("je.env.runCheckpointer", "false");
217         newConfig.setConfigParam("je.env.runCleaner", "false");
218         env.setMutableConfig(newConfig);
219 
220         final int N_RECS = 25;
221 
222         /* Intentionally corrupt the transaction commit record. */
223         CheckpointConfig chkConf = new CheckpointConfig();
224         chkConf.setForce(true);
225         Transaction txn = env.beginTransaction(null, null);
226         for (int i = 0; i < N_RECS; i++) {
227             String keyStr = Integer.toString(i);
228             DatabaseEntry key =
229                 new DatabaseEntry(StringUtils.toUTF8(keyStr));
230             DatabaseEntry data =
231                 new DatabaseEntry(new byte[100]);
232             try {
233                 assertTrue(db.put(txn, key, data) ==
234                            OperationStatus.SUCCESS);
235             } catch (DatabaseException DE) {
236                 fail("unexpected DatabaseException");
237                 break;
238             }
239             try {
240                 FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
241                 txn.commit();
242                 fail("expected DatabaseException");
243             } catch (DatabaseException DE) {
244             }
245             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
246             txn = env.beginTransaction(null, null);
247         }
248 
249         FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
250 
251         try {
252             txn.commit();
253         } catch (DatabaseException DE) {
254             fail("unexpected DatabaseException");
255         }
256 
257         /* Flush the corrupted records to disk. */
258         try {
259             env.checkpoint(chkConf);
260         } catch (DatabaseException DE) {
261             DE.printStackTrace();
262             fail("unexpected DatabaseException");
263         }
264 
265         EnvironmentStats stats = env.getStats(null);
266         assertTrue((stats.getNFullINFlush() +
267                     stats.getNFullBINFlush()) > 0);
268 
269         /*
270          * Figure out where the log starts and ends, and make a local
271          * FileReader class to mimic reading the logs. The only action we need
272          * is to run checksums on the entries.
273          */
274         EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
275         long lastLsn = envImpl.getFileManager().getLastUsedLsn();
276         Long firstFile = envImpl.getFileManager().getFirstFileNum();
277         long firstLsn = DbLsn.makeLsn(firstFile, 0);
278 
279         FileReader reader = new FileReader
280             (envImpl,
281              4096,              // readBufferSize
282              true,              // forward
283              firstLsn,
284              null,              // singleFileNumber
285              lastLsn,           // end of file lsn
286              DbLsn.NULL_LSN) {  // finishLsn
287 
288                 @Override
289                 protected boolean processEntry(ByteBuffer entryBuffer) {
290                     entryBuffer.position(entryBuffer.position() +
291                                          currentEntryHeader.getItemSize());
292                     return true;
293                 }
294             };
295 
296         /* Read the logs, checking checksums. */
297         while (reader.readNextEntry()) {
298         }
299 
300         /* Check that the reader reads all the way to last entry. */
301         assertEquals("last=" + DbLsn.getNoFormatString(lastLsn) +
302                      " readerlast=" +
303                      DbLsn.getNoFormatString(reader.getLastLsn()),
304                      lastLsn,
305                      reader.getLastLsn());
306     }
307 
308     @Test
testLogBufferOverflowAbortNoDupes()309     public void testLogBufferOverflowAbortNoDupes() {
310         doLogBufferOverflowTest(false, false);
311     }
312 
313     @Test
testLogBufferOverflowCommitNoDupes()314     public void testLogBufferOverflowCommitNoDupes() {
315         doLogBufferOverflowTest(true, false);
316     }
317 
318     @Test
testLogBufferOverflowAbortDupes()319     public void testLogBufferOverflowAbortDupes() {
320         doLogBufferOverflowTest(false, true);
321     }
322 
323     @Test
testLogBufferOverflowCommitDupes()324     public void testLogBufferOverflowCommitDupes() {
325         doLogBufferOverflowTest(true, true);
326     }
327 
doLogBufferOverflowTest(boolean abort, boolean dupes)328     private void doLogBufferOverflowTest(boolean abort, boolean dupes) {
329         try {
330             EnvironmentConfig envConfig = TestUtils.initEnvConfig();
331             envConfig.setTransactional(true);
332             envConfig.setAllowCreate(true);
333             envConfig.setCacheSize(100000);
334             env = new Environment(envHome, envConfig);
335 
336             String databaseName = "ioexceptiondb";
337             DatabaseConfig dbConfig = new DatabaseConfig();
338             dbConfig.setAllowCreate(true);
339             dbConfig.setSortedDuplicates(true);
340             dbConfig.setTransactional(true);
341             db = env.openDatabase(null, databaseName, dbConfig);
342 
343             Transaction txn = env.beginTransaction(null, null);
344             DatabaseEntry oneKey =
345                 (dupes ?
346                  new DatabaseEntry(StringUtils.toUTF8("2")) :
347                  new DatabaseEntry(StringUtils.toUTF8("1")));
348             DatabaseEntry oneData =
349                 new DatabaseEntry(new byte[10]);
350             DatabaseEntry twoKey =
351                 new DatabaseEntry(StringUtils.toUTF8("2"));
352             DatabaseEntry twoData =
353                 new DatabaseEntry(new byte[100000]);
354             if (dupes) {
355                 DatabaseEntry temp = oneKey;
356                 oneKey = oneData;
357                 oneData = temp;
358                 temp = twoKey;
359                 twoKey = twoData;
360                 twoData = temp;
361             }
362 
363             try {
364                 assertTrue(db.put(txn, oneKey, oneData) ==
365                            OperationStatus.SUCCESS);
366                 db.put(txn, twoKey, twoData);
367             } catch (DatabaseException DE) {
368                 fail("unexpected DatabaseException");
369             }
370 
371             /* Read back the data and make sure it all looks ok. */
372             try {
373                 assertTrue(db.get(txn, oneKey, oneData, null) ==
374                            OperationStatus.SUCCESS);
375                 assertTrue(oneData.getData().length == (dupes ? 1 : 10));
376             } catch (DatabaseException DE) {
377                 fail("unexpected DatabaseException");
378             }
379 
380             try {
381                 assertTrue(db.get(txn, twoKey, twoData, null) ==
382                            OperationStatus.SUCCESS);
383             } catch (DatabaseException DE) {
384                 fail("unexpected DatabaseException");
385             }
386 
387             try {
388                 if (abort) {
389                     txn.abort();
390                 } else {
391                     txn.commit();
392                 }
393             } catch (DatabaseException DE) {
394                 fail("unexpected DatabaseException");
395             }
396 
397             /* Read back the data and make sure it all looks ok. */
398             try {
399                 assertTrue(db.get(null, oneKey, oneData, null) ==
400                            (abort ?
401                             OperationStatus.NOTFOUND :
402                             OperationStatus.SUCCESS));
403                 assertTrue(oneData.getData().length == (dupes ? 1 : 10));
404             } catch (DatabaseException DE) {
405                 fail("unexpected DatabaseException");
406             }
407 
408             try {
409                 assertTrue(db.get(null, twoKey, twoData, null) ==
410                            (abort ?
411                             OperationStatus.NOTFOUND :
412                             OperationStatus.SUCCESS));
413             } catch (DatabaseException DE) {
414                 fail("unexpected DatabaseException");
415             }
416 
417         } catch (Exception E) {
418             E.printStackTrace();
419         }
420     }
421 
422     @Test
testPutTransactionalWithIOException()423     public void testPutTransactionalWithIOException() {
424         try {
425             createDatabase(100000, 0, true);
426 
427             Transaction txn = env.beginTransaction(null, null);
428             int keyInt = 0;
429             String keyStr;
430             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
431 
432             /* Fill up the buffer until we see an IOException. */
433             while (true) {
434                 keyStr = Integer.toString(++keyInt);
435                 DatabaseEntry key =
436                     new DatabaseEntry(StringUtils.toUTF8(keyStr));
437                 DatabaseEntry data =
438                     new DatabaseEntry(StringUtils.toUTF8(("d" + keyStr)));
439                 try {
440                     assertTrue(db.put(txn, key, data) ==
441                                OperationStatus.SUCCESS);
442                 } catch (DatabaseException DE) {
443                     break;
444                 }
445             }
446 
447             /* Buffer still hasn't been written.  This should also fail. */
448             try {
449                 db.put(txn,
450                        new DatabaseEntry(StringUtils.toUTF8("shouldFail")),
451                        new DatabaseEntry(StringUtils.toUTF8("shouldFailD")));
452                 fail("expected DatabaseException");
453             } catch (DatabaseException e) {
454                 /* Expected. */
455             }
456             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
457 
458             /* Fails because the txn is abort-only. */
459             try {
460                 db.put(txn,
461                        new DatabaseEntry(StringUtils.toUTF8("shouldAlsoFail")),
462                        new DatabaseEntry(StringUtils.toUTF8("shouldAlsoFailD")));
463                 fail("expected DatabaseException");
464             } catch (OperationFailureException e) {
465                 /* Expected. */
466             }
467             txn.abort();
468 
469             /* Txn aborted.  None of the entries should be found. */
470             DatabaseEntry data = new DatabaseEntry();
471             assertTrue(db.get(null,
472                               new DatabaseEntry
473                               (StringUtils.toUTF8("shouldAlsoFail")),
474                               data,
475                               null) == OperationStatus.NOTFOUND);
476 
477             assertTrue(db.get(null,
478                               new DatabaseEntry
479                               (StringUtils.toUTF8("shouldFail")),
480                               data,
481                               null) == OperationStatus.NOTFOUND);
482 
483             assertTrue(db.get(null,
484                               new DatabaseEntry
485                               (StringUtils.toUTF8("shouldFail")),
486                               data,
487                               null) == OperationStatus.NOTFOUND);
488 
489             assertTrue(db.get(null,
490                               new DatabaseEntry(StringUtils.toUTF8(keyStr)),
491                               data,
492                               null) == OperationStatus.NOTFOUND);
493 
494             for (int i = --keyInt; i > 0; i--) {
495                 keyStr = Integer.toString(i);
496                 assertTrue(db.get(null,
497                                   new DatabaseEntry(StringUtils.toUTF8(keyStr)),
498                                   data,
499                                   null) == OperationStatus.NOTFOUND);
500             }
501 
502         } catch (Throwable T) {
503             T.printStackTrace();
504         }
505     }
506 
507     @Test
testIOExceptionDuringFileFlippingWrite()508     public void testIOExceptionDuringFileFlippingWrite() {
509         doIOExceptionDuringFileFlippingWrite(8, 33, 2);
510     }
511 
doIOExceptionDuringFileFlippingWrite(int numIterations, int exceptionStartWrite, int exceptionWriteCount)512     private void doIOExceptionDuringFileFlippingWrite(int numIterations,
513                                                       int exceptionStartWrite,
514                                                       int exceptionWriteCount) {
515         try {
516             EnvironmentConfig envConfig = new EnvironmentConfig();
517             DbInternal.disableParameterValidation(envConfig);
518             envConfig.setTransactional(true);
519             envConfig.setAllowCreate(true);
520             envConfig.setConfigParam("je.log.fileMax", "1000");
521             envConfig.setConfigParam("je.log.bufferSize", "1025");
522             envConfig.setConfigParam("je.env.runCheckpointer", "false");
523             envConfig.setConfigParam("je.env.runCleaner", "false");
524             env = new Environment(envHome, envConfig);
525 
526             EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
527             DatabaseConfig dbConfig = new DatabaseConfig();
528             dbConfig.setTransactional(true);
529             dbConfig.setAllowCreate(true);
530             db = env.openDatabase(null, "foo", dbConfig);
531 
532             /*
533              * Put one record into the database so it gets populated w/INs and
534              * LNs, and we can fake out the RMW commits used below.
535              */
536             DatabaseEntry key = new DatabaseEntry();
537             DatabaseEntry data = new DatabaseEntry();
538             IntegerBinding.intToEntry(5, key);
539             IntegerBinding.intToEntry(5, data);
540             db.put(null, key, data);
541 
542             /*
543              * Now generate trace and commit log entries. The trace records
544              * aren't forced out, but the commit records are forced.
545              */
546             FileManager.WRITE_COUNT = 0;
547             FileManager.THROW_ON_WRITE = true;
548             FileManager.STOP_ON_WRITE_COUNT = exceptionStartWrite;
549             FileManager.N_BAD_WRITES = exceptionWriteCount;
550             for (int i = 0; i < numIterations; i++) {
551 
552                 try {
553                     /* Generate a non-forced record. */
554                     if (i == (numIterations - 1)) {
555 
556                         /*
557                          * On the last iteration, write a record that is large
558                          * enough to force a file flip (i.e. an fsync which
559                          * succeeds) followed by the large write (which doesn't
560                          * succeed due to an IOException).  In [#15754] the
561                          * large write fails on Out Of Disk Space, rolling back
562                          * the savedLSN to the previous file, even though the
563                          * file has flipped.  The subsequent write ends up in
564                          * the flipped file, but at the offset of the older
565                          * file (leaving a hole in the new flipped file).
566                          */
567                         Trace.trace(envImpl,
568                                     i + "/" + FileManager.WRITE_COUNT +
569                                     " " + new String(new byte[2000]));
570                     } else {
571                         Trace.trace(envImpl,
572                                         i + "/" + FileManager.WRITE_COUNT +
573                                         " " + "xx");
574                     }
575                 } catch (IllegalStateException ISE) {
576                     /* Eat exception thrown by TraceLogHandler. */
577                 }
578 
579                 /*
580                  * Generate a forced record by calling commit. Since RMW
581                  * transactions that didn't actually do a write won't log a
582                  * commit record, do an addLogInfo to trick the txn into
583                  * logging a commit.
584                  */
585                 Transaction txn = env.beginTransaction(null, null);
586                 db.get(txn, key, data, LockMode.RMW);
587                 DbInternal.getTxn(txn).addLogInfo(DbLsn.makeLsn(3, 3));
588                 txn.commit();
589             }
590             db.close();
591 
592             /*
593              * Verify that the log files are ok and have no checksum errors.
594              */
595             FileReader reader =
596                 new FileReader(DbInternal.getEnvironmentImpl(env),
597                                4096, true, 0, null, DbLsn.NULL_LSN,
598                                DbLsn.NULL_LSN) {
599                     @Override
600             protected boolean processEntry(ByteBuffer entryBuffer) {
601                         entryBuffer.position(entryBuffer.position() +
602                                              currentEntryHeader.getItemSize());
603                         return true;
604                     }
605                 };
606 
607             DbInternal.getEnvironmentImpl(env).getLogManager().flush();
608 
609             while (reader.readNextEntry()) {
610             }
611 
612             /* Make sure the reader really did scan the files. */
613             assert (DbLsn.getFileNumber(reader.getLastLsn()) == 3) :
614                 DbLsn.toString(reader.getLastLsn());
615 
616             env.close();
617             env = null;
618             db = null;
619         } catch (Throwable T) {
620             T.printStackTrace();
621         } finally {
622             FileManager.STOP_ON_WRITE_COUNT = Long.MAX_VALUE;
623             FileManager.N_BAD_WRITES = Long.MAX_VALUE;
624         }
625     }
626 
627     /*
628      * Test the following sequence:
629      *
630      * write LN, commit;
631      * write same LN (getting an IOException),
632      * write another LN (getting an IOException) verify fails due to must-abort
633      * either commit(should fail and abort automatically)
634      * or abort (should always succeed).
635      * Verify UP, ensuring that LSN of LN is not marked obsolete.
636      */
637     @Test
testSR15761Part1()638     public void testSR15761Part1() {
639         doSR15761Test(true);
640     }
641 
642     @Test
testSR15761Part2()643     public void testSR15761Part2() {
644         doSR15761Test(false);
645     }
646 
doSR15761Test(boolean doCommit)647     private void doSR15761Test(boolean doCommit) {
648         try {
649             createDatabase(100000, 0, false);
650 
651             Transaction txn = env.beginTransaction(null, null);
652             int keyInt = 0;
653             String keyStr;
654 
655             keyStr = Integer.toString(keyInt);
656             DatabaseEntry key = new DatabaseEntry(StringUtils.toUTF8(keyStr));
657             DatabaseEntry data = new DatabaseEntry(new byte[2888]);
658             try {
659                 assertTrue(db.put(txn, key, data) == OperationStatus.SUCCESS);
660             } catch (DatabaseException DE) {
661                 fail("should have completed");
662             }
663             txn.commit();
664 
665             EnvironmentStats stats = env.getStats(null);
666             int nLocksPrePut = stats.getNTotalLocks();
667             txn = env.beginTransaction(null, null);
668             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
669             try {
670                 data = new DatabaseEntry(new byte[10000]);
671                 assertTrue(db.put(txn, key, data) == OperationStatus.SUCCESS);
672                 fail("expected IOException");
673             } catch (DatabaseException DE) {
674                 /* Expected */
675             }
676 
677             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
678             try {
679                 data = new DatabaseEntry(new byte[10]);
680                 assertTrue(db.put(txn, key, data) == OperationStatus.SUCCESS);
681                 fail("expected IOException");
682             } catch (OperationFailureException ISE) {
683                 /* Expected */
684             }
685 
686             FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
687             if (doCommit) {
688                 try {
689                     txn.commit();
690                     fail("expected must-abort transaction exception");
691                 } catch (OperationFailureException DE) {
692                     /* Expected. */
693                 }
694             }
695             try {
696                 txn.abort();
697             } catch (DatabaseException DE) {
698                 fail("expected abort to succeed");
699             }
700 
701             /* Lock should not be held. */
702             stats = env.getStats(null);
703             int nLocksPostPut = stats.getNTotalLocks();
704             assertTrue(nLocksPrePut == nLocksPostPut);
705 
706             UtilizationProfile up =
707                 DbInternal.getEnvironmentImpl(env).getUtilizationProfile();
708 
709             /*
710              * Checkpoint the environment to flush all utilization tracking
711              * information before verifying.
712              */
713             CheckpointConfig ckptConfig = new CheckpointConfig();
714             ckptConfig.setForce(true);
715             env.checkpoint(ckptConfig);
716 
717             assertTrue(up.verifyFileSummaryDatabase());
718         } catch (Throwable T) {
719             T.printStackTrace();
720         }
721     }
722 
723     @Test
testAbortWithIOException()724     public void testAbortWithIOException()
725         throws Throwable {
726 
727         Transaction txn = null;
728         createDatabase(0, 0, true);
729         writeAndVerify(null, false, "k1", "d1", false);
730         writeAndVerify(null, true, "k2", "d2", false);
731         writeAndVerify(null, false, "k3", "d3", false);
732 
733         FileManager.IO_EXCEPTION_TESTING_ON_WRITE = true;
734         EnvironmentStats stats = env.getStats(null);
735         int nLocksPreGet = stats.getNTotalLocks();
736 
737         /* Loop doing aborts until the buffer fills up and we get an IOE. */
738         int keySuffix = 1;
739         boolean done = false;
740         while (!done) {
741             txn = env.beginTransaction(null, null);
742 
743             DatabaseEntry key =
744                 new DatabaseEntry(StringUtils.toUTF8(("key" + keySuffix)));
745             DatabaseEntry data = key;
746             try {
747                 OperationStatus status = db.put(txn, key, data);
748                 assertTrue(status == (OperationStatus.SUCCESS));
749 
750                  stats = env.getStats(null);
751             } catch (Exception e) {
752                 done = true;
753             }
754 
755             try {
756                    txn.abort();
757 
758                 /*
759                  * Keep going until we actually get an IOException from the
760                  * buffer filling up.
761                  */
762                 continue;
763             } catch (DatabaseException DE) {
764                 done = true;
765             }
766         }
767 
768         FileManager.IO_EXCEPTION_TESTING_ON_WRITE = false;
769 
770         /* Lock should not be held. */
771         stats = env.getStats(null);
772         int nLocksPostAbort = stats.getNTotalLocks();
773         assertTrue(nLocksPreGet == nLocksPostAbort);
774     }
775 
doIOExceptionTest(boolean doRecovery)776     private void doIOExceptionTest(boolean doRecovery)
777         throws Throwable {
778 
779         Transaction txn = null;
780         createDatabase(0, 0, true);
781         writeAndVerify(null, false, "k1", "d1", doRecovery);
782         writeAndVerify(null, true, "k2", "d2", doRecovery);
783         writeAndVerify(null, false, "k3", "d3", doRecovery);
784 
785         txn = env.beginTransaction(null, null);
786         writeAndVerify(txn, false, "k4", "d4", false);
787         txn.abort();
788         verify(null, true, "k4", doRecovery);
789         verify(null, false, "k1", doRecovery);
790         verify(null, false, "k3", doRecovery);
791 
792         txn = env.beginTransaction(null, null);
793         writeAndVerify(txn, false, "k4", "d4", false);
794         txn.commit();
795         verify(null, false, "k4", doRecovery);
796 
797         txn = env.beginTransaction(null, null);
798         writeAndVerify(txn, true, "k5", "d5", false);
799         /* Ensure that writes after IOExceptions don't succeed. */
800         writeAndVerify(txn, false, "k5a", "d5a", false);
801         txn.abort();
802         verify(null, true, "k5", doRecovery);
803         verify(null, true, "k5a", doRecovery);
804 
805         txn = env.beginTransaction(null, null);
806 
807         EnvironmentStats stats = env.getStats(null);
808         int nLocksPrePut = stats.getNTotalLocks();
809 
810         writeAndVerify(txn, false, "k6", "d6", false);
811         writeAndVerify(txn, true, "k6a", "d6a", false);
812 
813         stats = env.getStats(null);
814         try {
815             txn.commit();
816             fail("expected DatabaseException");
817         } catch (DatabaseException DE) {
818         }
819 
820         /* Lock should not be held. */
821         stats = env.getStats(null);
822         int nLocksPostCommit = stats.getNTotalLocks();
823         assertTrue(nLocksPrePut == nLocksPostCommit);
824 
825         verify(null, true, "k6", doRecovery);
826         verify(null, true, "k6a", doRecovery);
827 
828         txn = env.beginTransaction(null, null);
829         writeAndVerify(txn, false, "k6", "d6", false);
830         writeAndVerify(txn, true, "k6a", "d6a", false);
831         writeAndVerify(txn, false, "k6b", "d6b", false);
832 
833         try {
834             txn.commit();
835         } catch (DatabaseException DE) {
836             fail("expected success");
837         }
838 
839         /*
840          * k6a will still exist because the writeAndVerify didn't fail -- there
841          * was no write.  The write happens at commit time.
842          */
843         verify(null, false, "k6", doRecovery);
844         verify(null, false, "k6a", doRecovery);
845         verify(null, false, "k6b", doRecovery);
846     }
847 
writeAndVerify(Transaction txn, boolean throwIOException, String keyString, String dataString, boolean doRecovery)848     private void writeAndVerify(Transaction txn,
849                                 boolean throwIOException,
850                                 String keyString,
851                                 String dataString,
852                                 boolean doRecovery)
853         throws DatabaseException {
854 
855         DatabaseEntry key = new DatabaseEntry(StringUtils.toUTF8(keyString));
856         DatabaseEntry data = new DatabaseEntry(StringUtils.toUTF8(dataString));
857         FileManager.IO_EXCEPTION_TESTING_ON_WRITE = throwIOException;
858         try {
859             assertTrue(db.put(txn, key, data) == OperationStatus.SUCCESS);
860 
861             /*
862              * We don't expect an IOException if we're in a transaction because
863              * the put() only writes to the buffer, not the disk.  The write to
864              * disk doesn't happen until the commit/abort.
865              */
866             if (throwIOException && txn == null) {
867                 fail("didn't catch DatabaseException.");
868             }
869         } catch (DatabaseException DE) {
870             if (!throwIOException || txn != null) {
871                 fail("caught DatabaseException.");
872             }
873         }
874         verify(txn, throwIOException, keyString, doRecovery);
875     }
876 
verify(Transaction txn, boolean expectFailure, String keyString, boolean doRecovery)877     private void verify(Transaction txn,
878                         boolean expectFailure,
879                         String keyString,
880                         boolean doRecovery)
881         throws DatabaseException {
882 
883         if (doRecovery) {
884             db.close();
885             forceCloseEnvOnly();
886             createDatabase(0, 0, true);
887         }
888         DatabaseEntry key = new DatabaseEntry(StringUtils.toUTF8(keyString));
889         DatabaseEntry returnedData = new DatabaseEntry();
890         OperationStatus status =
891             db.get(txn,
892                    key,
893                    returnedData,
894                    LockMode.DEFAULT);
895         assertTrue("status " + status + " txn " + (txn != null),
896                    status == ((expectFailure && txn == null) ?
897                               OperationStatus.NOTFOUND :
898                               OperationStatus.SUCCESS));
899     }
900 
createDatabase(long cacheSize, long maxFileSize, boolean dups)901     private void createDatabase(long cacheSize, long maxFileSize, boolean dups)
902         throws DatabaseException {
903 
904         EnvironmentConfig envConfig = TestUtils.initEnvConfig();
905         envConfig.setTransactional(true);
906         envConfig.setAllowCreate(true);
907         envConfig.setConfigParam
908             (EnvironmentParams.NUM_LOG_BUFFERS.getName(), "2");
909         envConfig.setConfigParam
910             (EnvironmentParams.LOG_MEM_SIZE.getName(),
911              EnvironmentParams.LOG_MEM_SIZE_MIN_STRING);
912         if (maxFileSize != 0) {
913             DbInternal.disableParameterValidation(envConfig);
914             envConfig.setConfigParam
915                 (EnvironmentParams.LOG_FILE_MAX.getName(), "" + maxFileSize);
916         }
917         if (cacheSize != 0) {
918             envConfig.setCacheSize(cacheSize);
919         }
920         env = new Environment(envHome, envConfig);
921 
922         String databaseName = "ioexceptiondb";
923         DatabaseConfig dbConfig = new DatabaseConfig();
924         dbConfig.setAllowCreate(true);
925         dbConfig.setSortedDuplicates(dups);
926         dbConfig.setTransactional(true);
927         db = env.openDatabase(null, databaseName, dbConfig);
928     }
929 
930     /* Force the environment to be closed even with outstanding handles.*/
forceCloseEnvOnly()931     private void forceCloseEnvOnly()
932         throws DatabaseException {
933 
934         /* Close w/out checkpointing, in order to exercise recovery better.*/
935         try {
936             DbInternal.getEnvironmentImpl(env).close(false);
937         } catch (DatabaseException DE) {
938             if (!FileManager.IO_EXCEPTION_TESTING_ON_WRITE) {
939                 throw DE;
940             } else {
941                 /* Expect an exception from flushing the log manager. */
942             }
943         }
944         env = null;
945     }
946 }
947