hbase/regionserver/TestRegionReplicas.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hbase.regionserver;

import static org.apache.hadoop.hbase.regionserver.TestRegionServerNoMaster.*;
import java.io.IOException;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TestMetaTableAccessor;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.util.StringUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import com.google.protobuf.ServiceException;

/**
 * Tests for region replicas. Sad that we cannot isolate these without bringing up a whole
 * cluster. See {@link TestRegionServerNoMaster}.
 */
@Category(MediumTests.class)
public class TestRegionReplicas {
  private static final Log LOG = LogFactory.getLog(TestRegionReplicas.class);

  private static final int NB_SERVERS = 1;
  private static HTable table;
  private static final byte[] row = "TestRegionReplicas".getBytes();

  private static HRegionInfo hriPrimary;
  private static HRegionInfo hriSecondary;

  private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
  private static final byte[] f = HConstants.CATALOG_FAMILY;

  @BeforeClass
  public static void before() throws Exception {
    // Reduce the hdfs block size and prefetch to trigger the file-link reopen
    // when the file is moved to archive (e.g. compaction)
    HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192);
    HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1);
    HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024);

    HTU.startMiniCluster(NB_SERVERS);
    final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName());

    // Create table then get the single region for our new table.
    table = HTU.createTable(tableName, f);

    hriPrimary = table.getRegionLocation(row, false).getRegionInfo();

    // mock a secondary region info to open
    hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(),
        hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1);

    // No master
    TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU);
  }

  @AfterClass
  public static void afterClass() throws Exception {
    table.close();
    HTU.shutdownMiniCluster();
  }

  @After
  public void after() throws Exception {
    // Clean the state if the test failed before cleaning the znode
    // It does not manage all bad failures, so if there are multiple failures, only
    //  the first one should be looked at.
    ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary);
  }

  private HRegionServer getRS() {
    return HTU.getMiniHBaseCluster().getRegionServer(0);
  }

  @Test(timeout = 60000)
  public void testOpenRegionReplica() throws Exception {
    openRegion(HTU, getRS(), hriSecondary);
    try {
      //load some data to primary
      HTU.loadNumericRows(table, f, 0, 1000);

      // assert that we can read back from primary
      Assert.assertEquals(1000, HTU.countRows(table));
    } finally {
      HTU.deleteNumericRows(table, f, 0, 1000);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  /** Tests that the meta location is saved for secondary regions */
  @Test(timeout = 60000)
  public void testRegionReplicaUpdatesMetaLocation() throws Exception {
    openRegion(HTU, getRS(), hriSecondary);
    Table meta = null;
    try {
      meta = HTU.getConnection().getTable(TableName.META_TABLE_NAME);
      TestMetaTableAccessor.assertMetaLocation(meta, hriPrimary.getRegionName()
        , getRS().getServerName(), -1, 1, false);
    } finally {
      if (meta != null ) meta.close();
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  @Test(timeout = 60000)
  public void testRegionReplicaGets() throws Exception {
    try {
      //load some data to primary
      HTU.loadNumericRows(table, f, 0, 1000);
      // assert that we can read back from primary
      Assert.assertEquals(1000, HTU.countRows(table));
      // flush so that region replica can read
      Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);

      openRegion(HTU, getRS(), hriSecondary);

      // first try directly against region
      region = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
      assertGet(region, 42, true);

      assertGetRpc(hriSecondary, 42, true);
    } finally {
      HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  @Test(timeout = 60000)
  public void testGetOnTargetRegionReplica() throws Exception {
    try {
      //load some data to primary
      HTU.loadNumericRows(table, f, 0, 1000);
      // assert that we can read back from primary
      Assert.assertEquals(1000, HTU.countRows(table));
      // flush so that region replica can read
      Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);

      openRegion(HTU, getRS(), hriSecondary);

      // try directly Get against region replica
      byte[] row = Bytes.toBytes(String.valueOf(42));
      Get get = new Get(row);
      get.setConsistency(Consistency.TIMELINE);
      get.setReplicaId(1);
      Result result = table.get(get);
      Assert.assertArrayEquals(row, result.getValue(f, null));
    } finally {
      HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  private void assertGet(Region region, int value, boolean expect) throws IOException {
    byte[] row = Bytes.toBytes(String.valueOf(value));
    Get get = new Get(row);
    Result result = region.get(get);
    if (expect) {
      Assert.assertArrayEquals(row, result.getValue(f, null));
    } else {
      result.isEmpty();
    }
  }

  // build a mock rpc
  private void assertGetRpc(HRegionInfo info, int value, boolean expect)
      throws IOException, ServiceException {
    byte[] row = Bytes.toBytes(String.valueOf(value));
    Get get = new Get(row);
    ClientProtos.GetRequest getReq = RequestConverter.buildGetRequest(info.getRegionName(), get);
    ClientProtos.GetResponse getResp =  getRS().getRSRpcServices().get(null, getReq);
    Result result = ProtobufUtil.toResult(getResp.getResult());
    if (expect) {
      Assert.assertArrayEquals(row, result.getValue(f, null));
    } else {
      result.isEmpty();
    }
  }

  private void restartRegionServer() throws Exception {
    afterClass();
    before();
  }

  @Test(timeout = 300000)
  public void testRefreshStoreFiles() throws Exception {
    // enable store file refreshing
    final int refreshPeriod = 2000; // 2 sec
    HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 100);
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
      refreshPeriod);
    // restart the region server so that it starts the refresher chore
    restartRegionServer();

    try {
      LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
      openRegion(HTU, getRS(), hriSecondary);

      //load some data to primary
      LOG.info("Loading data to primary region");
      HTU.loadNumericRows(table, f, 0, 1000);
      // assert that we can read back from primary
      Assert.assertEquals(1000, HTU.countRows(table));
      // flush so that region replica can read
      LOG.info("Flushing primary region");
      Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);

      // ensure that chore is run
      LOG.info("Sleeping for " + (4 * refreshPeriod));
      Threads.sleep(4 * refreshPeriod);

      LOG.info("Checking results from secondary region replica");
      Region secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
      Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());

      assertGet(secondaryRegion, 42, true);
      assertGetRpc(hriSecondary, 42, true);
      assertGetRpc(hriSecondary, 1042, false);

      //load some data to primary
      HTU.loadNumericRows(table, f, 1000, 1100);
      region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);

      HTU.loadNumericRows(table, f, 2000, 2100);
      region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);

      // ensure that chore is run
      Threads.sleep(4 * refreshPeriod);

      assertGetRpc(hriSecondary, 42, true);
      assertGetRpc(hriSecondary, 1042, true);
      assertGetRpc(hriSecondary, 2042, true);

      // ensure that we are see the 3 store files
      Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());

      // force compaction
      HTU.compact(table.getName(), true);

      long wakeUpTime = System.currentTimeMillis() + 4 * refreshPeriod;
      while (System.currentTimeMillis() < wakeUpTime) {
        assertGetRpc(hriSecondary, 42, true);
        assertGetRpc(hriSecondary, 1042, true);
        assertGetRpc(hriSecondary, 2042, true);
        Threads.sleep(10);
      }

      // ensure that we see the compacted file only
      Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());

    } finally {
      HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  @Test(timeout = 300000)
  public void testFlushAndCompactionsInPrimary() throws Exception {

    long runtime = 30 * 1000;
    // enable store file refreshing
    final int refreshPeriod = 100; // 100ms refresh is a lot
    HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 3);
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
    // restart the region server so that it starts the refresher chore
    restartRegionServer();
    final int startKey = 0, endKey = 1000;

    try {
      openRegion(HTU, getRS(), hriSecondary);

      //load some data to primary so that reader won't fail
      HTU.loadNumericRows(table, f, startKey, endKey);
      TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
      // ensure that chore is run
      Threads.sleep(2 * refreshPeriod);

      final AtomicBoolean running = new AtomicBoolean(true);
      @SuppressWarnings("unchecked")
      final AtomicReference<Exception>[] exceptions = new AtomicReference[3];
      for (int i=0; i < exceptions.length; i++) {
        exceptions[i] = new AtomicReference<Exception>();
      }

      Runnable writer = new Runnable() {
        int key = startKey;
        @Override
        public void run() {
          try {
            while (running.get()) {
              byte[] data = Bytes.toBytes(String.valueOf(key));
              Put put = new Put(data);
              put.add(f, null, data);
              table.put(put);
              key++;
              if (key == endKey) key = startKey;
            }
          } catch (Exception ex) {
            LOG.warn(ex);
            exceptions[0].compareAndSet(null, ex);
          }
        }
      };

      Runnable flusherCompactor = new Runnable() {
        Random random = new Random();
        @Override
        public void run() {
          try {
            while (running.get()) {
              // flush or compact
              if (random.nextBoolean()) {
                TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
              } else {
                HTU.compact(table.getName(), random.nextBoolean());
              }
            }
          } catch (Exception ex) {
            LOG.warn(ex);
            exceptions[1].compareAndSet(null, ex);
          }
        }
      };

      Runnable reader = new Runnable() {
        Random random = new Random();
        @Override
        public void run() {
          try {
            while (running.get()) {
              // whether to do a close and open
              if (random.nextInt(10) == 0) {
                try {
                  closeRegion(HTU, getRS(), hriSecondary);
                } catch (Exception ex) {
                  LOG.warn("Failed closing the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
                  exceptions[2].compareAndSet(null, ex);
                }
                try {
                  openRegion(HTU, getRS(), hriSecondary);
                } catch (Exception ex) {
                  LOG.warn("Failed opening the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
                  exceptions[2].compareAndSet(null, ex);
                }
              }

              int key = random.nextInt(endKey - startKey) + startKey;
              assertGetRpc(hriSecondary, key, true);
            }
          } catch (Exception ex) {
            LOG.warn("Failed getting the value in the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
            exceptions[2].compareAndSet(null, ex);
          }
        }
      };

      LOG.info("Starting writer and reader");
      ExecutorService executor = Executors.newFixedThreadPool(3);
      executor.submit(writer);
      executor.submit(flusherCompactor);
      executor.submit(reader);

      // wait for threads
      Threads.sleep(runtime);
      running.set(false);
      executor.shutdown();
      executor.awaitTermination(30, TimeUnit.SECONDS);

      for (AtomicReference<Exception> exRef : exceptions) {
        Assert.assertNull(exRef.get());
      }
    } finally {
      HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }

  @Test(timeout = 300000)
  public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
    // disable the store file refresh chore (we do this by hand)
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
    restartRegionServer();

    try {
      LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
      openRegion(HTU, getRS(), hriSecondary);

      // load some data to primary
      LOG.info("Loading data to primary region");
      for (int i = 0; i < 3; ++i) {
        HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
        Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
        region.flush(true);
      }

      Region primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName());
      Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());

      // Refresh store files on the secondary
      Region secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
      secondaryRegion.getStore(f).refreshStoreFiles();
      Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());

      // force compaction
      LOG.info("Force Major compaction on primary region " + hriPrimary);
      primaryRegion.compact(true);
      Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());

      // scan all the hfiles on the secondary.
      // since there are no read on the secondary when we ask locations to
      // the NN a FileNotFound exception will be returned and the FileLink
      // should be able to deal with it giving us all the result we expect.
      int keys = 0;
      int sum = 0;
      for (StoreFile sf: secondaryRegion.getStore(f).getStorefiles()) {
        // Our file does not exist anymore. was moved by the compaction above.
        LOG.debug(getRS().getFileSystem().exists(sf.getPath()));
        Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));

        HFileScanner scanner = sf.getReader().getScanner(false, false);
        scanner.seekTo();
        do {
          keys++;

          Cell cell = scanner.getKeyValue();
          sum += Integer.parseInt(Bytes.toString(cell.getRowArray(),
            cell.getRowOffset(), cell.getRowLength()));
        } while (scanner.next());
      }
      Assert.assertEquals(3000, keys);
      Assert.assertEquals(4498500, sum);
    } finally {
      HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
      closeRegion(HTU, getRS(), hriSecondary);
    }
  }
}