1 /*- 2 * See the file LICENSE for redistribution information. 3 * 4 * Copyright (c) 2002, 2014 Oracle and/or its affiliates. All rights reserved. 5 * 6 */ 7 8 package com.sleepycat.je.rep.impl.networkRestore; 9 10 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.DISPOSED_COUNT; 11 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.FETCH_COUNT; 12 import static com.sleepycat.je.rep.impl.networkRestore.NetworkBackupStatDefinition.SKIP_COUNT; 13 import static org.junit.Assert.assertEquals; 14 import static org.junit.Assert.assertNull; 15 import static org.junit.Assert.assertTrue; 16 import static org.junit.Assert.fail; 17 18 import java.io.File; 19 import java.io.FileInputStream; 20 import java.io.FileOutputStream; 21 import java.io.IOException; 22 import java.net.InetSocketAddress; 23 import java.util.Arrays; 24 import java.util.List; 25 26 import org.junit.After; 27 import org.junit.Before; 28 import org.junit.Test; 29 import org.junit.runner.RunWith; 30 import org.junit.runners.Parameterized; 31 import org.junit.runners.Parameterized.Parameters; 32 33 import com.sleepycat.bind.tuple.IntegerBinding; 34 import com.sleepycat.bind.tuple.LongBinding; 35 import com.sleepycat.je.Database; 36 import com.sleepycat.je.DatabaseConfig; 37 import com.sleepycat.je.DatabaseEntry; 38 import com.sleepycat.je.DatabaseException; 39 import com.sleepycat.je.DbInternal; 40 import com.sleepycat.je.Environment; 41 import com.sleepycat.je.EnvironmentConfig; 42 import com.sleepycat.je.EnvironmentFailureException; 43 import com.sleepycat.je.EnvironmentLockedException; 44 import com.sleepycat.je.VerifyConfig; 45 import com.sleepycat.je.dbi.EnvironmentFailureReason; 46 import com.sleepycat.je.log.FileManager; 47 import com.sleepycat.je.rep.ReplicationNetworkConfig; 48 import com.sleepycat.je.rep.impl.node.NameIdPair; 49 import com.sleepycat.je.rep.net.DataChannelFactory; 50 import com.sleepycat.je.rep.utilint.BinaryProtocol.ProtocolException; 51 import com.sleepycat.je.rep.utilint.net.DataChannelFactoryBuilder; 52 import com.sleepycat.je.rep.utilint.RepTestUtils; 53 import com.sleepycat.je.rep.utilint.ServiceDispatcher; 54 import com.sleepycat.je.util.DbBackup; 55 import com.sleepycat.je.util.TestUtils; 56 import com.sleepycat.util.test.SharedTestUtils; 57 import com.sleepycat.util.test.TestBase; 58 59 @RunWith(Parameterized.class) 60 public class NetworkBackupTest extends TestBase { 61 62 /* The port being handled by the dispatcher. */ 63 private static final int TEST_PORT = 5000; 64 65 private File envHome; 66 private EnvironmentConfig envConfig; 67 File backupDir; 68 private Environment env; 69 private Environment backupEnv; 70 private FileManager fileManager; 71 private Database db; 72 73 private final InetSocketAddress serverAddress = 74 new InetSocketAddress("localhost", TEST_PORT); 75 76 private DataChannelFactory channelFactory; 77 private ServiceDispatcher dispatcher; 78 private FeederManager fm; 79 80 protected DatabaseConfig dbconfig; 81 protected final DatabaseEntry key = new DatabaseEntry(new byte[] { 1 }); 82 protected final DatabaseEntry data = new DatabaseEntry(new byte[] { 100 }); 83 protected static final String TEST_DB_NAME = "TestDB"; 84 85 protected static final VerifyConfig vconfig = new VerifyConfig(); 86 87 private static final int DB_ENTRIES = 100; 88 89 static { 90 vconfig.setAggressive(false); 91 vconfig.setPropagateExceptions(true); 92 } 93 94 /* True if the Feeder enables multiple sub directories. */ 95 private boolean envMultiDirs; 96 97 /* 98 * True if the nodes need to copy log files enables multiple sub 99 * directories. 100 */ 101 private boolean backupMultiDirs; 102 private final int DATA_DIRS = 3; 103 104 /* 105 * Experiences four cases: 106 * 1. Feeder doesn't enable sub directories, nor replicas. 107 * 2. Feeder doesn't enable sub directories, but replicas do. 108 * 3. Feeder enables sub directories, but replicas don't. 109 * 4. Feeder enables sub directories, so do replicas. 110 */ 111 @Parameters genParams()112 public static List<Object[]> genParams() { 113 114 return Arrays.asList(new Object[][] {{false, false}, {false, true}, 115 {true, false}, {true, true}}); 116 } 117 NetworkBackupTest(boolean envMultiDirs, boolean backupMultiDirs)118 public NetworkBackupTest(boolean envMultiDirs, boolean backupMultiDirs) { 119 this.envMultiDirs = envMultiDirs; 120 this.backupMultiDirs = backupMultiDirs; 121 customName = (envMultiDirs ? ":env-multi-sub-dirs" : "") + 122 (backupMultiDirs ? ":backup-multi-sub-dirs" : ""); 123 } 124 125 @Before setUp()126 public void setUp() 127 throws Exception { 128 129 super.setUp(); 130 envHome = SharedTestUtils.getTestDir(); 131 envConfig = TestUtils.initEnvConfig(); 132 DbInternal.disableParameterValidation(envConfig); 133 envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, "1000"); 134 envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false"); 135 136 /* If multiple sub directories property is enabled. */ 137 if (envMultiDirs) { 138 envConfig.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, 139 DATA_DIRS + ""); 140 createSubDir(envHome, false); 141 } 142 envConfig.setAllowCreate(true); 143 envConfig.setTransactional(true); 144 145 env = new Environment(envHome, envConfig); 146 147 dbconfig = new DatabaseConfig(); 148 dbconfig.setAllowCreate(true); 149 dbconfig.setTransactional(true); 150 dbconfig.setSortedDuplicates(false); 151 db = env.openDatabase(null, TEST_DB_NAME, dbconfig); 152 153 for (int i = 0; i < DB_ENTRIES; i++) { 154 IntegerBinding.intToEntry(i, key); 155 LongBinding.longToEntry(i, data); 156 db.put(null, key, data); 157 } 158 /* Create cleaner fodder. */ 159 for (int i = 0; i < (DB_ENTRIES / 2); i++) { 160 IntegerBinding.intToEntry(i, key); 161 LongBinding.longToEntry(i, data); 162 db.put(null, key, data); 163 } 164 env.cleanLog(); 165 env.verify(vconfig, System.err); 166 167 /* Create the backup environment. */ 168 backupDir = new File(envHome.getCanonicalPath() + ".backup"); 169 /* Clear the log files in the backup directory. */ 170 cleanEnvHome(backupDir, true); 171 /* Create the Ennvironment home for replicas. */ 172 if (backupMultiDirs) { 173 envConfig.setConfigParam(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, 174 DATA_DIRS + ""); 175 createSubDir(backupDir, true); 176 } else { 177 envConfig.setConfigParam 178 (EnvironmentConfig.LOG_N_DATA_DIRECTORIES, "0"); 179 backupDir.mkdir(); 180 } 181 assertTrue(backupDir.exists()); 182 183 backupEnv = new Environment(backupDir, envConfig); 184 fileManager = 185 DbInternal.getEnvironmentImpl(backupEnv).getFileManager(); 186 187 final ReplicationNetworkConfig repNetConfig = 188 ReplicationNetworkConfig.create(RepTestUtils.readNetProps()); 189 channelFactory = DataChannelFactoryBuilder.construct(repNetConfig); 190 191 dispatcher = new ServiceDispatcher(serverAddress, channelFactory); 192 dispatcher.start(); 193 fm = new FeederManager(dispatcher, 194 DbInternal.getEnvironmentImpl(env), 195 new NameIdPair("n1", (short) 1)); 196 fm.start(); 197 } 198 createSubDir(File home, boolean isBackupDir)199 private void createSubDir(File home, boolean isBackupDir) 200 throws Exception { 201 202 if (isBackupDir) { 203 if (!home.exists()) { 204 home.mkdir(); 205 } 206 } 207 208 if ((envMultiDirs && !isBackupDir) || 209 (backupMultiDirs && isBackupDir)) { 210 for (int i = 1; i <= DATA_DIRS; i++) { 211 File subDir = new File(home, TestUtils.getSubDirName(i)); 212 assertTrue(!subDir.exists()); 213 assertTrue(subDir.mkdir()); 214 } 215 } 216 } 217 218 @After tearDown()219 public void tearDown() 220 throws Exception { 221 222 try { 223 db.close(); 224 env.close(); 225 fm.shutdown(); 226 dispatcher.shutdown(); 227 } catch (Exception e) { 228 e.printStackTrace(); 229 throw e; 230 } 231 } 232 cleanEnvHome(File home, boolean isBackupDir)233 private void cleanEnvHome(File home, boolean isBackupDir) 234 throws Exception { 235 236 if (home == null) { 237 return; 238 } 239 240 File[] files = home.listFiles(); 241 if (files == null || files.length == 0) { 242 return; 243 } 244 245 /* Delete the sub directories if any. */ 246 for (File file : files) { 247 if (file.isDirectory() && file.getName().startsWith("data")) { 248 File[] subFiles = file.listFiles(); 249 for (File subFile : subFiles) { 250 assertTrue(subFile.delete()); 251 } 252 assertTrue(file.delete()); 253 } 254 255 if (isBackupDir && file.isFile()) { 256 assertTrue(file.delete()); 257 } 258 } 259 260 TestUtils.removeLogFiles("tearDown", home, false); 261 262 if (isBackupDir) { 263 assertTrue(home.delete()); 264 } 265 } 266 267 @Test testBackupFiles()268 public void testBackupFiles() 269 throws Exception { 270 271 /* The client side */ 272 NetworkBackup backup1 = 273 new NetworkBackup(serverAddress, 274 backupEnv.getHome(), 275 new NameIdPair("n1", (short) 1), 276 false, 277 fileManager, 278 channelFactory); 279 String files1[] = backup1.execute(); 280 assertEquals(0, backup1.getStats().getInt(SKIP_COUNT)); 281 282 verify(envHome, backupDir, files1); 283 284 /* Corrupt the currently backed up log files. */ 285 for (File f : fileManager.listJDBFiles()) { 286 FileOutputStream os = new FileOutputStream(f); 287 os.write(1); 288 os.close(); 289 } 290 int count = fileManager.listJDBFiles().length; 291 NetworkBackup backup2 = 292 new NetworkBackup(serverAddress, 293 backupEnv.getHome(), 294 new NameIdPair("n1", (short) 1), 295 false, 296 fileManager, 297 channelFactory); 298 String files2[] = backup2.execute(); 299 verify(envHome, backupDir, files2); 300 assertEquals(count, backup2.getStats().getInt(DISPOSED_COUNT)); 301 302 verifyAsEnv(backupDir); 303 304 /* 305 * Close the database to avoid problems later when we corrupt the files 306 * on the server 307 */ 308 db.close(); 309 310 /* Corrupt files on the server, and make sure no files are copied */ 311 for (final File f : 312 DbInternal.getEnvironmentImpl(env).getFileManager(). 313 listJDBFiles()) { 314 final FileOutputStream os = new FileOutputStream(f); 315 os.write(1); 316 os.close(); 317 } 318 final NetworkBackup backup3 = 319 new NetworkBackup(serverAddress, 320 backupEnv.getHome(), 321 new NameIdPair("n1", (short) 1), 322 true, 323 fileManager, 324 channelFactory); 325 try { 326 backup3.execute(); 327 fail("Expected IOException"); 328 } catch (IOException e) { 329 } 330 331 assertEquals("No files should have been fetched", 332 0, 333 backup3.getStats().getInt(FETCH_COUNT)); 334 335 /* The environment is corrupted -- invalidate it */ 336 new EnvironmentFailureException( 337 DbInternal.getEnvironmentImpl(env), 338 EnvironmentFailureReason.TEST_INVALIDATE); 339 } 340 341 /** 342 * Performs a backup while the database is growing actively 343 * 344 * @throws InterruptedException 345 * @throws IOException 346 * @throws DatabaseException 347 */ 348 @Test testConcurrentBackup()349 public void testConcurrentBackup() 350 throws InterruptedException, IOException, DatabaseException { 351 352 LogFileGeneratingThread lfThread = new LogFileGeneratingThread(); 353 BackupThread backupThread = new BackupThread(); 354 lfThread.start(); 355 356 backupThread.start(); 357 backupThread.join(60*1000); 358 lfThread.quit = true; 359 lfThread.join(60*1000); 360 361 DbBackup dbBackup = new DbBackup(env); 362 dbBackup.startBackup(); 363 int newCount = dbBackup.getLogFilesInBackupSet().length; 364 365 assertNull(backupThread.error); 366 assertNull(lfThread.error); 367 368 /* 369 * Verify that the count did increase while the backup was in progress. 370 */ 371 assertTrue(newCount > backupThread.files.length); 372 /* Verify that the backup was correct. */ 373 verify(envHome, backupDir, backupThread.files); 374 375 verifyAsEnv(backupDir); 376 dbBackup.endBackup(); 377 } 378 379 class BackupThread extends Thread { 380 Exception error = null; 381 String files[] = null; 382 BackupThread()383 BackupThread() { 384 setDaemon(true); 385 } 386 387 @Override run()388 public void run() { 389 try { 390 NetworkBackup backup1 = 391 new NetworkBackup(serverAddress, 392 backupEnv.getHome(), 393 new NameIdPair("n1", (short) 1), 394 true, 395 fileManager, 396 channelFactory); 397 files = backup1.execute(); 398 } catch (Exception e) { 399 error = e; 400 error.printStackTrace(); 401 } 402 } 403 } 404 405 class LogFileGeneratingThread extends Thread { 406 Exception error = null; 407 volatile boolean quit = false; 408 LogFileGeneratingThread()409 LogFileGeneratingThread() { 410 setDaemon(true); 411 } 412 413 @Override run()414 public void run() { 415 try { 416 for (int i = 0; i < 100000; i++) { 417 IntegerBinding.intToEntry(i, key); 418 LongBinding.longToEntry(i, data); 419 db.put(null, key, data); 420 if (quit) { 421 return; 422 } 423 } 424 } catch (Exception e) { 425 error = e; 426 error.printStackTrace(); 427 } 428 fail("Backup did not finish in time"); 429 } 430 } 431 432 @Test testBasic()433 public void testBasic() 434 throws Exception { 435 436 /* The client side */ 437 NetworkBackup backup1 = 438 new NetworkBackup(serverAddress, 439 backupEnv.getHome(), 440 new NameIdPair("n1", (short) 1), 441 true, 442 fileManager, 443 channelFactory); 444 backup1.execute(); 445 assertEquals(0, backup1.getStats().getInt(SKIP_COUNT)); 446 447 /* 448 * repeat, should find mostly cached files. Invoking backup causes 449 * a checkpoint to be written to the log. 450 */ 451 NetworkBackup backup2 = 452 new NetworkBackup(serverAddress, 453 backupEnv.getHome(), 454 new NameIdPair("n1", (short) 1), 455 true, 456 fileManager, 457 channelFactory); 458 String files2[] = backup2.execute(); 459 verify(envHome, backupDir, files2); 460 461 assertTrue((backup1.getStats().getInt(FETCH_COUNT) - 462 backup2.getStats().getInt(SKIP_COUNT)) <= 1); 463 464 verifyAsEnv(backupDir); 465 } 466 467 @Test testLeaseBasic()468 public void testLeaseBasic() 469 throws Exception { 470 471 int errorFileNum = 2; 472 NetworkBackup backup1 = 473 new TestNetworkBackup(serverAddress, 474 backupEnv, 475 (short) 1, 476 true, 477 errorFileNum); 478 try { 479 backup1.execute(); 480 fail("Exception expected"); 481 } catch (IOException e) { 482 /* Expected. */ 483 } 484 /* Wait for server to detect a broken connection. */ 485 Thread.sleep(500); 486 /* Verify that the lease was created. */ 487 assertEquals(1, fm.getLeaseCount()); 488 NetworkBackup backup2 = 489 new NetworkBackup(serverAddress, 490 backupEnv.getHome(), 491 new NameIdPair("n1", (short) 1), 492 true, 493 fileManager, 494 channelFactory); 495 /* Verify that the lease was renewed. */ 496 String[] files2 = backup2.execute(); 497 assertEquals(2, backup2.getStats().getInt(SKIP_COUNT)); 498 assertEquals(1, fm.getLeaseRenewalCount()); 499 500 /* Verify that the copy resumed correctly. */ 501 verify(envHome, backupDir, files2); 502 503 verifyAsEnv(backupDir); 504 } 505 506 @Test testLeaseExpiration()507 public void testLeaseExpiration() 508 throws Exception { 509 510 int errorFileNum = 2; 511 512 /* 513 * Verify that leases are created and expire as expected. 514 */ 515 NetworkBackup backup1 = new TestNetworkBackup(serverAddress, 516 backupEnv, 517 (short) 1, 518 true, 519 errorFileNum); 520 /* Shorten the lease duration for test purposes. */ 521 long leaseDuration = 1*1000; 522 try { 523 fm.setLeaseDuration(leaseDuration); 524 backup1.execute(); 525 fail("Exception expected"); 526 } catch (IOException e) { 527 /* Expected. */ 528 } 529 /* Wait for server to detect broken connection. */ 530 Thread.sleep(500); 531 /* Verify that the lease was created. */ 532 assertEquals(1, fm.getLeaseCount()); 533 Thread.sleep(leaseDuration); 534 /* Verify that the lease has expired after its duration. */ 535 assertEquals(0, fm.getLeaseCount()); 536 537 /* Resume after lease expiration. */ 538 NetworkBackup backup2 = 539 new NetworkBackup(serverAddress, 540 backupEnv.getHome(), 541 new NameIdPair("n1", (short) 1), 542 true, 543 fileManager, 544 channelFactory); 545 /* Verify that the lease was renewed. */ 546 String[] files2 = backup2.execute(); 547 /* Verify that the copy resumed correctly. */ 548 verify(envHome, backupDir, files2); 549 550 verifyAsEnv(backupDir); 551 } 552 verify(File envDir, File envBackupDir, String backupEnvFiles[])553 private void verify(File envDir, 554 File envBackupDir, 555 String backupEnvFiles[]) 556 throws IOException { 557 558 for (String backupFile : backupEnvFiles) { 559 File envFile = null; 560 561 /* 562 * The file names returned by NetworkBackup only apply on the 563 * replicas (if they have sub directories enabled while Feeder 564 * doesn't), so need to calculate the real path on the Feeder 565 * according the file name. 566 */ 567 if (envMultiDirs) { 568 if (backupMultiDirs) { 569 envFile = new File(envDir, backupFile); 570 } else { 571 envFile = new File(DbInternal.getEnvironmentImpl(env). 572 getFileManager(). 573 getFullFileName(backupFile)); 574 } 575 } else { 576 if (backupMultiDirs) { 577 int start = backupFile.indexOf(File.separator); 578 envFile = new File 579 (envDir, 580 backupFile.substring(start, backupFile.length())); 581 } else { 582 envFile = new File(envDir, backupFile); 583 } 584 } 585 FileInputStream envStream = new FileInputStream(envFile); 586 FileInputStream envBackupStream = 587 new FileInputStream(new File(envBackupDir, backupFile)); 588 int ib1, ib2; 589 do { 590 ib1 = envStream.read(); 591 ib2 = envBackupStream.read(); 592 } while ((ib1 == ib2) && (ib1 != -1)); 593 assertEquals(ib1, ib2); 594 envStream.close(); 595 envBackupStream.close(); 596 } 597 } 598 verifyAsEnv(File dir)599 void verifyAsEnv(File dir) 600 throws EnvironmentLockedException, DatabaseException { 601 602 /* Close the backupEnv abnormally. */ 603 DbInternal.getEnvironmentImpl(backupEnv).abnormalClose(); 604 605 Environment benv = new Environment(dir, envConfig); 606 /* Note that verify modifies log files. */ 607 benv.verify(vconfig, System.err); 608 benv.close(); 609 } 610 cleanBackupdir()611 private void cleanBackupdir() { 612 for (File f : backupDir.listFiles()) { 613 assertTrue(f.delete()); 614 } 615 } 616 617 /** 618 * Class to provoke a client failure when requesting a specific file. 619 */ 620 private class TestNetworkBackup extends NetworkBackup { 621 int errorFileNum = 0; 622 TestNetworkBackup(InetSocketAddress serverSocket, Environment backupEnv, short clientId, boolean retainLogfiles, int errorFileNum)623 public TestNetworkBackup(InetSocketAddress serverSocket, 624 Environment backupEnv, 625 short clientId, 626 boolean retainLogfiles, 627 int errorFileNum) 628 throws DatabaseException { 629 630 super(serverSocket, 631 backupEnv.getHome(), 632 new NameIdPair("node"+clientId, clientId), 633 retainLogfiles, 634 DbInternal.getEnvironmentImpl(backupEnv).getFileManager(), 635 channelFactory); 636 this.errorFileNum = errorFileNum; 637 } 638 639 @Override getFile(File file)640 protected void getFile(File file) 641 throws IOException, ProtocolException, DigestException { 642 if (errorFileNum-- == 0) { 643 throw new IOException("test exception"); 644 } 645 super.getFile(file); 646 } 647 } 648 } 649