1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.rep.impl.networkRestore;
9 
10 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.DISPOSED_COUNT;
11 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.FETCH_COUNT;
12 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.SKIP_COUNT;
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertNull;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 
18 import java.io.File;
19 import java.io.FileInputStream;
20 import java.io.FileOutputStream;
21 import java.io.IOException;
22 import java.net.InetSocketAddress;
23 import java.util.Arrays;
24 import java.util.List;
25 
26 import org.junit.After;
27 import org.junit.Before;
28 import org.junit.Test;
29 import org.junit.runner.RunWith;
30 import org.junit.runners.Parameterized;
31 import org.junit.runners.Parameterized.Parameters;
32 
33 import com.sleepycat.bind.tuple.IntegerBinding;
34 import com.sleepycat.bind.tuple.LongBinding;
35 import com.sleepycat.je.Database;
36 import com.sleepycat.je.DatabaseConfig;
37 import com.sleepycat.je.DatabaseEntry;
38 import com.sleepycat.je.DatabaseException;
39 import com.sleepycat.je.DbInternal;
40 import com.sleepycat.je.Environment;
41 import com.sleepycat.je.EnvironmentConfig;
42 import com.sleepycat.je.EnvironmentFailureException;
43 import com.sleepycat.je.EnvironmentLockedException;
44 import com.sleepycat.je.VerifyConfig;
45 import com.sleepycat.je.dbi.EnvironmentFailureReason;
46 import com.sleepycat.je.log.FileManager;
47 import com.sleepycat.je.rep.ReplicationNetworkConfig;
48 import com.sleepycat.je.rep.impl.node.NameIdPair;
49 import com.sleepycat.je.rep.net.DataChannelFactory;
50 import com.sleepycat.je.rep.utilint.BinaryProtocol.ProtocolException;
51 import com.sleepycat.je.rep.utilint.net.DataChannelFactoryBuilder;
52 import com.sleepycat.je.rep.utilint.RepTestUtils;
53 import com.sleepycat.je.rep.utilint.ServiceDispatcher;
54 import com.sleepycat.je.util.DbBackup;
55 import com.sleepycat.je.util.TestUtils;
56 import com.sleepycat.util.test.SharedTestUtils;
57 import com.sleepycat.util.test.TestBase;
58 
59 @RunWith(Parameterized.class)
60 public class NetworkBackupTest extends TestBase {
61 
62     /* The port being handled by the dispatcher. */
63     private static final int TEST_PORT = 5000;
64 
65     private File envHome;
66     private EnvironmentConfig envConfig;
67     File backupDir;
68     private Environment env;
69     private Environment backupEnv;
70     private FileManager fileManager;
71     private Database db;
72 
73     private final InetSocketAddress serverAddress =
74         new InetSocketAddress("localhost", TEST_PORT);
75 
76     private DataChannelFactory channelFactory;
77     private ServiceDispatcher dispatcher;
78     private FeederManager fm;
79 
80     protected DatabaseConfig dbconfig;
81     protected final DatabaseEntry key = new DatabaseEntry(new byte[] { 1 });
82     protected final DatabaseEntry data = new DatabaseEntry(new byte[] { 100 });
83     protected static final String TEST_DB_NAME = "TestDB";
84 
85     protected static final VerifyConfig vconfig = new VerifyConfig();
86 
87     private static final int DB_ENTRIES = 100;
88 
89     static {
90         vconfig.setAggressive(false);
91         vconfig.setPropagateExceptions(true);
92     }
93 
94     /* True if the Feeder enables multiple sub directories. */
95     private boolean envMultiDirs;
96 
97     /*
98      * True if the nodes need to copy log files enables multiple sub
99      * directories.
100      */
101     private boolean backupMultiDirs;
102     private final int DATA_DIRS = 3;
103 
104     /*
105      * Experiences four cases:
106      * 1. Feeder doesn't enable sub directories, nor replicas.
107      * 2. Feeder doesn't enable sub directories, but replicas do.
108      * 3. Feeder enables sub directories, but replicas don't.
109      * 4. Feeder enables sub directories, so do replicas.
110      */
111     @Parameters
genParams()112     public static List<Object[]> genParams() {
113 
114         return Arrays.asList(new Object[][] {{false, false}, {false, true},
115             {true, false}, {true, true}});
116     }
117 
NetworkBackupTest(boolean envMultiDirs, boolean backupMultiDirs)118     public NetworkBackupTest(boolean envMultiDirs, boolean backupMultiDirs) {
119         this.envMultiDirs = envMultiDirs;
120         this.backupMultiDirs = backupMultiDirs;
121         customName = (envMultiDirs ? ":env-multi-sub-dirs" : "") +
122                 (backupMultiDirs ? ":backup-multi-sub-dirs" : "");
123     }
124 
125     @Before
setUp()126     public void setUp()
127         throws Exception {
128 
129         super.setUp();
130         envHome = SharedTestUtils.getTestDir();
131         envConfig = TestUtils.initEnvConfig();
132         DbInternal.disableParameterValidation(envConfig);
133         envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, "1000");
134         envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
135 
136         /* If multiple sub directories property is enabled. */
137         if (envMultiDirs) {
138             envConfig.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES,
139                                      DATA_DIRS + "");
140             createSubDir(envHome, false);
141         }
142         envConfig.setAllowCreate(true);
143         envConfig.setTransactional(true);
144 
145         env = new Environment(envHome, envConfig);
146 
147         dbconfig = new DatabaseConfig();
148         dbconfig.setAllowCreate(true);
149         dbconfig.setTransactional(true);
150         dbconfig.setSortedDuplicates(false);
151         db = env.openDatabase(null, TEST_DB_NAME, dbconfig);
152 
153         for (int i = 0; i < DB_ENTRIES; i++) {
154             IntegerBinding.intToEntry(i, key);
155             LongBinding.longToEntry(i, data);
156             db.put(null, key, data);
157         }
158         /* Create cleaner fodder. */
159         for (int i = 0; i < (DB_ENTRIES / 2); i++) {
160             IntegerBinding.intToEntry(i, key);
161             LongBinding.longToEntry(i, data);
162             db.put(null, key, data);
163         }
164         env.cleanLog();
165         env.verify(vconfig, System.err);
166 
167         /* Create the backup environment. */
168         backupDir = new File(envHome.getCanonicalPath() + ".backup");
169         /* Clear the log files in the backup directory. */
170         cleanEnvHome(backupDir, true);
171         /* Create the Ennvironment home for replicas. */
172         if (backupMultiDirs) {
173             envConfig.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES,
174                                      DATA_DIRS + "");
175             createSubDir(backupDir, true);
176         } else {
177             envConfig.setConfigParam
178                 (EnvironmentConfig.LOG_N_DATA_DIRECTORIES, "0");
179             backupDir.mkdir();
180         }
181         assertTrue(backupDir.exists());
182 
183         backupEnv = new Environment(backupDir, envConfig);
184         fileManager =
185             DbInternal.getEnvironmentImpl(backupEnv).getFileManager();
186 
187         final ReplicationNetworkConfig repNetConfig =
188             ReplicationNetworkConfig.create(RepTestUtils.readNetProps());
189         channelFactory = DataChannelFactoryBuilder.construct(repNetConfig);
190 
191         dispatcher = new ServiceDispatcher(serverAddress, channelFactory);
192         dispatcher.start();
193         fm = new FeederManager(dispatcher,
194                                DbInternal.getEnvironmentImpl(env),
195                                new NameIdPair("n1", (short) 1));
196         fm.start();
197     }
198 
createSubDir(File home, boolean isBackupDir)199     private void createSubDir(File home, boolean isBackupDir)
200         throws Exception {
201 
202         if (isBackupDir) {
203             if (!home.exists()) {
204                 home.mkdir();
205             }
206         }
207 
208         if ((envMultiDirs && !isBackupDir) ||
209             (backupMultiDirs && isBackupDir)) {
210             for (int i = 1; i <= DATA_DIRS; i++) {
211                 File subDir = new File(home, TestUtils.getSubDirName(i));
212                 assertTrue(!subDir.exists());
213                 assertTrue(subDir.mkdir());
214             }
215         }
216     }
217 
218     @After
tearDown()219     public void tearDown()
220         throws Exception {
221 
222         try {
223             db.close();
224             env.close();
225             fm.shutdown();
226             dispatcher.shutdown();
227         } catch (Exception e) {
228             e.printStackTrace();
229             throw e;
230         }
231     }
232 
cleanEnvHome(File home, boolean isBackupDir)233     private void cleanEnvHome(File home, boolean isBackupDir)
234         throws Exception {
235 
236         if (home == null) {
237             return;
238         }
239 
240         File[] files = home.listFiles();
241         if (files == null || files.length == 0) {
242             return;
243         }
244 
245         /* Delete the sub directories if any. */
246         for (File file : files) {
247             if (file.isDirectory() && file.getName().startsWith("data")) {
248                 File[] subFiles = file.listFiles();
249                 for (File subFile : subFiles) {
250                     assertTrue(subFile.delete());
251                 }
252                 assertTrue(file.delete());
253             }
254 
255             if (isBackupDir && file.isFile()) {
256                 assertTrue(file.delete());
257             }
258         }
259 
260         TestUtils.removeLogFiles("tearDown", home, false);
261 
262         if (isBackupDir) {
263             assertTrue(home.delete());
264         }
265     }
266 
267     @Test
testBackupFiles()268     public void testBackupFiles()
269         throws Exception {
270 
271         /* The client side */
272         NetworkBackup backup1 =
273             new NetworkBackup(serverAddress,
274                               backupEnv.getHome(),
275                               new NameIdPair("n1", (short) 1),
276                               false,
277                               fileManager,
278                               channelFactory);
279         String files1[] = backup1.execute();
280         assertEquals(0, backup1.getStats().getInt(SKIP_COUNT));
281 
282         verify(envHome, backupDir, files1);
283 
284         /* Corrupt the currently backed up log files. */
285         for (File f : fileManager.listJDBFiles()) {
286             FileOutputStream os = new FileOutputStream(f);
287             os.write(1);
288             os.close();
289         }
290         int count = fileManager.listJDBFiles().length;
291         NetworkBackup backup2 =
292             new NetworkBackup(serverAddress,
293                               backupEnv.getHome(),
294                               new NameIdPair("n1", (short) 1),
295                               false,
296                               fileManager,
297                               channelFactory);
298         String files2[] = backup2.execute();
299         verify(envHome, backupDir, files2);
300         assertEquals(count, backup2.getStats().getInt(DISPOSED_COUNT));
301 
302         verifyAsEnv(backupDir);
303 
304         /*
305          * Close the database to avoid problems later when we corrupt the files
306          * on the server
307          */
308         db.close();
309 
310         /* Corrupt files on the server, and make sure no files are copied */
311         for (final File f :
312                  DbInternal.getEnvironmentImpl(env).getFileManager().
313                  listJDBFiles()) {
314             final FileOutputStream os = new FileOutputStream(f);
315             os.write(1);
316             os.close();
317         }
318         final NetworkBackup backup3 =
319             new NetworkBackup(serverAddress,
320                               backupEnv.getHome(),
321                               new NameIdPair("n1", (short) 1),
322                               true,
323                               fileManager,
324                               channelFactory);
325         try {
326             backup3.execute();
327             fail("Expected IOException");
328         } catch (IOException e) {
329         }
330 
331         assertEquals("No files should have been fetched",
332                      0,
333                      backup3.getStats().getInt(FETCH_COUNT));
334 
335         /* The environment is corrupted -- invalidate it */
336         new EnvironmentFailureException(
337             DbInternal.getEnvironmentImpl(env),
338             EnvironmentFailureReason.TEST_INVALIDATE);
339     }
340 
341     /**
342      * Performs a backup while the database is growing actively
343      *
344      * @throws InterruptedException
345      * @throws IOException
346      * @throws DatabaseException
347      */
348     @Test
testConcurrentBackup()349     public void testConcurrentBackup()
350         throws InterruptedException, IOException, DatabaseException {
351 
352         LogFileGeneratingThread lfThread = new LogFileGeneratingThread();
353         BackupThread backupThread = new BackupThread();
354         lfThread.start();
355 
356         backupThread.start();
357         backupThread.join(60*1000);
358         lfThread.quit = true;
359         lfThread.join(60*1000);
360 
361         DbBackup dbBackup = new DbBackup(env);
362         dbBackup.startBackup();
363         int newCount = dbBackup.getLogFilesInBackupSet().length;
364 
365         assertNull(backupThread.error);
366         assertNull(lfThread.error);
367 
368         /*
369          * Verify that the count did increase while the backup was in progress.
370          */
371         assertTrue(newCount > backupThread.files.length);
372         /* Verify that the backup was correct. */
373         verify(envHome, backupDir, backupThread.files);
374 
375         verifyAsEnv(backupDir);
376         dbBackup.endBackup();
377     }
378 
379     class BackupThread extends Thread {
380         Exception error = null;
381         String files[] = null;
382 
BackupThread()383         BackupThread() {
384             setDaemon(true);
385         }
386 
387         @Override
run()388         public void run() {
389             try {
390                 NetworkBackup backup1 =
391                     new NetworkBackup(serverAddress,
392                                       backupEnv.getHome(),
393                                       new NameIdPair("n1", (short) 1),
394                                       true,
395                                       fileManager,
396                                       channelFactory);
397                 files = backup1.execute();
398             } catch (Exception e) {
399                 error = e;
400                 error.printStackTrace();
401             }
402         }
403     }
404 
405     class LogFileGeneratingThread extends Thread {
406         Exception error = null;
407         volatile boolean quit = false;
408 
LogFileGeneratingThread()409         LogFileGeneratingThread() {
410             setDaemon(true);
411         }
412 
413         @Override
run()414         public void run() {
415             try {
416                 for (int i = 0; i < 100000; i++) {
417                     IntegerBinding.intToEntry(i, key);
418                     LongBinding.longToEntry(i, data);
419                     db.put(null, key, data);
420                     if (quit) {
421                         return;
422                     }
423                 }
424             } catch (Exception e) {
425                 error = e;
426                 error.printStackTrace();
427             }
428             fail("Backup did not finish in time");
429         }
430     }
431 
432     @Test
testBasic()433     public void testBasic()
434         throws Exception {
435 
436         /* The client side */
437         NetworkBackup backup1 =
438             new NetworkBackup(serverAddress,
439                               backupEnv.getHome(),
440                               new NameIdPair("n1", (short) 1),
441                               true,
442                               fileManager,
443                               channelFactory);
444         backup1.execute();
445         assertEquals(0, backup1.getStats().getInt(SKIP_COUNT));
446 
447         /*
448          * repeat, should find mostly cached files. Invoking backup causes
449          * a checkpoint to be written to the log.
450          */
451         NetworkBackup backup2 =
452             new NetworkBackup(serverAddress,
453                               backupEnv.getHome(),
454                               new NameIdPair("n1", (short) 1),
455                               true,
456                               fileManager,
457                               channelFactory);
458         String files2[] = backup2.execute();
459         verify(envHome, backupDir, files2);
460 
461         assertTrue((backup1.getStats().getInt(FETCH_COUNT) -
462                      backup2.getStats().getInt(SKIP_COUNT))  <= 1);
463 
464         verifyAsEnv(backupDir);
465     }
466 
467     @Test
testLeaseBasic()468     public void testLeaseBasic()
469         throws Exception {
470 
471         int errorFileNum = 2;
472         NetworkBackup backup1 =
473             new TestNetworkBackup(serverAddress,
474                                   backupEnv,
475                                   (short) 1,
476                                   true,
477                                   errorFileNum);
478         try {
479             backup1.execute();
480             fail("Exception expected");
481         } catch (IOException e) {
482             /* Expected. */
483         }
484         /* Wait for server to detect a broken connection. */
485         Thread.sleep(500);
486         /* Verify that the lease was created. */
487         assertEquals(1, fm.getLeaseCount());
488         NetworkBackup backup2 =
489             new NetworkBackup(serverAddress,
490                               backupEnv.getHome(),
491                               new NameIdPair("n1", (short) 1),
492                               true,
493                               fileManager,
494                               channelFactory);
495         /* Verify that the lease was renewed. */
496         String[] files2 = backup2.execute();
497         assertEquals(2, backup2.getStats().getInt(SKIP_COUNT));
498         assertEquals(1, fm.getLeaseRenewalCount());
499 
500         /* Verify that the copy resumed correctly. */
501         verify(envHome, backupDir, files2);
502 
503         verifyAsEnv(backupDir);
504     }
505 
506     @Test
testLeaseExpiration()507     public void testLeaseExpiration()
508         throws Exception {
509 
510         int errorFileNum = 2;
511 
512         /*
513          * Verify that leases are created and expire as expected.
514          */
515         NetworkBackup backup1 = new TestNetworkBackup(serverAddress,
516                                                       backupEnv,
517                                                       (short) 1,
518                                                       true,
519                                                       errorFileNum);
520         /* Shorten the lease duration for test purposes. */
521         long leaseDuration = 1*1000;
522         try {
523             fm.setLeaseDuration(leaseDuration);
524             backup1.execute();
525             fail("Exception expected");
526         } catch (IOException e) {
527             /* Expected. */
528         }
529         /* Wait for server to detect broken connection. */
530         Thread.sleep(500);
531         /* Verify that the lease was created. */
532         assertEquals(1, fm.getLeaseCount());
533         Thread.sleep(leaseDuration);
534         /* Verify that the lease has expired after its duration. */
535         assertEquals(0, fm.getLeaseCount());
536 
537         /* Resume after lease expiration. */
538         NetworkBackup backup2 =
539             new NetworkBackup(serverAddress,
540                               backupEnv.getHome(),
541                               new NameIdPair("n1", (short) 1),
542                               true,
543                               fileManager,
544                               channelFactory);
545         /* Verify that the lease was renewed. */
546         String[] files2 = backup2.execute();
547         /* Verify that the copy resumed correctly. */
548         verify(envHome, backupDir, files2);
549 
550         verifyAsEnv(backupDir);
551     }
552 
verify(File envDir, File envBackupDir, String backupEnvFiles[])553     private void verify(File envDir,
554                         File envBackupDir,
555                         String backupEnvFiles[])
556        throws IOException {
557 
558        for (String backupFile : backupEnvFiles) {
559            File envFile = null;
560 
561            /*
562             * The file names returned by NetworkBackup only apply on the
563             * replicas (if they have sub directories enabled while Feeder
564             * doesn't), so need to calculate the real path on the Feeder
565             * according the file name.
566             */
567            if (envMultiDirs) {
568                if (backupMultiDirs) {
569                    envFile = new File(envDir, backupFile);
570                } else {
571                    envFile = new File(DbInternal.getEnvironmentImpl(env).
572                                       getFileManager().
573                                       getFullFileName(backupFile));
574                }
575            } else {
576                if (backupMultiDirs) {
577                    int start = backupFile.indexOf(File.separator);
578                    envFile = new File
579                        (envDir,
580                         backupFile.substring(start, backupFile.length()));
581                } else {
582                    envFile = new File(envDir, backupFile);
583                }
584            }
585            FileInputStream envStream = new FileInputStream(envFile);
586            FileInputStream envBackupStream =
587                new FileInputStream(new File(envBackupDir, backupFile));
588            int ib1, ib2;
589            do {
590                ib1 = envStream.read();
591                ib2 = envBackupStream.read();
592            } while ((ib1 == ib2) && (ib1 != -1));
593            assertEquals(ib1, ib2);
594            envStream.close();
595            envBackupStream.close();
596        }
597     }
598 
verifyAsEnv(File dir)599     void verifyAsEnv(File dir)
600         throws EnvironmentLockedException, DatabaseException {
601 
602         /* Close the backupEnv abnormally. */
603         DbInternal.getEnvironmentImpl(backupEnv).abnormalClose();
604 
605         Environment benv = new Environment(dir, envConfig);
606         /* Note that verify modifies log files. */
607         benv.verify(vconfig, System.err);
608         benv.close();
609     }
610 
cleanBackupdir()611     private void cleanBackupdir() {
612         for (File f : backupDir.listFiles()) {
613             assertTrue(f.delete());
614         }
615     }
616 
617     /**
618      * Class to provoke a client failure when requesting a specific file.
619      */
620     private class TestNetworkBackup extends NetworkBackup {
621         int errorFileNum = 0;
622 
TestNetworkBackup(InetSocketAddress serverSocket, Environment backupEnv, short clientId, boolean retainLogfiles, int errorFileNum)623         public TestNetworkBackup(InetSocketAddress serverSocket,
624                                  Environment backupEnv,
625                                  short clientId,
626                                  boolean retainLogfiles,
627                                  int errorFileNum)
628             throws DatabaseException {
629 
630             super(serverSocket,
631                   backupEnv.getHome(),
632                   new NameIdPair("node"+clientId, clientId),
633                   retainLogfiles,
634                   DbInternal.getEnvironmentImpl(backupEnv).getFileManager(),
635                   channelFactory);
636             this.errorFileNum = errorFileNum;
637         }
638 
639         @Override
getFile(File file)640         protected void getFile(File file)
641             throws IOException, ProtocolException, DigestException {
642             if (errorFileNum-- == 0) {
643                 throw new IOException("test exception");
644             }
645             super.getFile(file);
646         }
647     }
648 }
649