1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with this 4 * work for additional information regarding copyright ownership. The ASF 5 * licenses this file to you under the Apache License, Version 2.0 (the 6 * "License"); you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 * License for the specific language governing permissions and limitations 15 * under the License. 16 */ 17 package org.apache.hadoop.hbase.client; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertTrue; 21 22 import java.util.ArrayList; 23 import java.util.List; 24 25 import org.apache.commons.logging.Log; 26 import org.apache.commons.logging.LogFactory; 27 import org.apache.hadoop.conf.Configuration; 28 import org.apache.hadoop.hbase.Cell; 29 import org.apache.hadoop.hbase.HBaseTestingUtility; 30 import org.apache.hadoop.hbase.HConstants; 31 import org.apache.hadoop.hbase.HRegionInfo; 32 import org.apache.hadoop.hbase.HRegionLocation; 33 import org.apache.hadoop.hbase.HTestConst; 34 import org.apache.hadoop.hbase.KeyValue; 35 import org.apache.hadoop.hbase.testclassification.MediumTests; 36 import org.apache.hadoop.hbase.MiniHBaseCluster; 37 import org.apache.hadoop.hbase.TableName; 38 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; 39 import org.apache.hadoop.hbase.filter.ColumnRangeFilter; 40 import org.apache.hadoop.hbase.master.HMaster; 41 import org.apache.hadoop.hbase.master.RegionState.State; 42 import org.apache.hadoop.hbase.master.RegionStates; 43 import org.apache.hadoop.hbase.protobuf.ProtobufUtil; 44 import org.apache.hadoop.hbase.regionserver.HRegionServer; 45 import org.apache.hadoop.hbase.util.Bytes; 46 import org.apache.hadoop.hbase.util.ConfigUtil; 47 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 48 import org.apache.hadoop.hbase.zookeeper.ZKAssign; 49 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; 50 import org.junit.After; 51 import org.junit.AfterClass; 52 import org.junit.Before; 53 import org.junit.BeforeClass; 54 import org.junit.Test; 55 import org.junit.experimental.categories.Category; 56 57 /** 58 * A client-side test, mostly testing scanners with various parameters. 59 */ 60 @Category(MediumTests.class) 61 public class TestScannersFromClientSide { 62 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class); 63 64 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 65 private static byte [] ROW = Bytes.toBytes("testRow"); 66 private static byte [] FAMILY = Bytes.toBytes("testFamily"); 67 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); 68 private static byte [] VALUE = Bytes.toBytes("testValue"); 69 70 /** 71 * @throws java.lang.Exception 72 */ 73 @BeforeClass setUpBeforeClass()74 public static void setUpBeforeClass() throws Exception { 75 Configuration conf = TEST_UTIL.getConfiguration(); 76 conf.setLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, 10 * 1024 * 1024); 77 TEST_UTIL.startMiniCluster(3); 78 } 79 80 /** 81 * @throws java.lang.Exception 82 */ 83 @AfterClass tearDownAfterClass()84 public static void tearDownAfterClass() throws Exception { 85 TEST_UTIL.shutdownMiniCluster(); 86 } 87 88 /** 89 * @throws java.lang.Exception 90 */ 91 @Before setUp()92 public void setUp() throws Exception { 93 // Nothing to do. 94 } 95 96 /** 97 * @throws java.lang.Exception 98 */ 99 @After tearDown()100 public void tearDown() throws Exception { 101 // Nothing to do. 102 } 103 104 /** 105 * Test from client side for batch of scan 106 * 107 * @throws Exception 108 */ 109 @Test testScanBatch()110 public void testScanBatch() throws Exception { 111 TableName TABLE = TableName.valueOf("testScanBatch"); 112 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8); 113 114 Table ht = TEST_UTIL.createTable(TABLE, FAMILY); 115 116 Put put; 117 Scan scan; 118 Delete delete; 119 Result result; 120 ResultScanner scanner; 121 boolean toLog = true; 122 List<Cell> kvListExp; 123 124 // table: row, family, c0:0, c1:1, ... , c7:7 125 put = new Put(ROW); 126 for (int i=0; i < QUALIFIERS.length; i++) { 127 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE); 128 put.add(kv); 129 } 130 ht.put(put); 131 132 // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7 133 put = new Put(ROW); 134 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE); 135 put.add(kv); 136 ht.put(put); 137 138 // delete upto ts: 3 139 delete = new Delete(ROW); 140 delete.deleteFamily(FAMILY, 3); 141 ht.delete(delete); 142 143 // without batch 144 scan = new Scan(ROW); 145 scan.setMaxVersions(); 146 scanner = ht.getScanner(scan); 147 148 // c4:4, c5:5, c6:6, c7:7 149 kvListExp = new ArrayList<Cell>(); 150 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE)); 151 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE)); 152 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE)); 153 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE)); 154 result = scanner.next(); 155 verifyResult(result, kvListExp, toLog, "Testing first batch of scan"); 156 157 // with batch 158 scan = new Scan(ROW); 159 scan.setMaxVersions(); 160 scan.setBatch(2); 161 scanner = ht.getScanner(scan); 162 163 // First batch: c4:4, c5:5 164 kvListExp = new ArrayList<Cell>(); 165 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE)); 166 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE)); 167 result = scanner.next(); 168 verifyResult(result, kvListExp, toLog, "Testing first batch of scan"); 169 170 // Second batch: c6:6, c7:7 171 kvListExp = new ArrayList<Cell>(); 172 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE)); 173 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE)); 174 result = scanner.next(); 175 verifyResult(result, kvListExp, toLog, "Testing second batch of scan"); 176 177 } 178 179 @Test testMaxResultSizeIsSetToDefault()180 public void testMaxResultSizeIsSetToDefault() throws Exception { 181 TableName TABLE = TableName.valueOf("testMaxResultSizeIsSetToDefault"); 182 Table ht = TEST_UTIL.createTable(TABLE, FAMILY); 183 184 // The max result size we expect the scan to use by default. 185 long expectedMaxResultSize = 186 TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, 187 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); 188 189 int numRows = 5; 190 byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); 191 192 int numQualifiers = 10; 193 byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); 194 195 // Specify the cell size such that a single row will be larger than the default 196 // value of maxResultSize. This means that Scan RPCs should return at most a single 197 // result back to the client. 198 int cellSize = (int) (expectedMaxResultSize / (numQualifiers - 1)); 199 byte[] cellValue = Bytes.createMaxByteArray(cellSize); 200 201 Put put; 202 List<Put> puts = new ArrayList<Put>(); 203 for (int row = 0; row < ROWS.length; row++) { 204 put = new Put(ROWS[row]); 205 for (int qual = 0; qual < QUALIFIERS.length; qual++) { 206 KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue); 207 put.add(kv); 208 } 209 puts.add(put); 210 } 211 ht.put(puts); 212 213 // Create a scan with the default configuration. 214 Scan scan = new Scan(); 215 216 ResultScanner scanner = ht.getScanner(scan); 217 assertTrue(scanner instanceof ClientScanner); 218 ClientScanner clientScanner = (ClientScanner) scanner; 219 220 // Call next to issue a single RPC to the server 221 scanner.next(); 222 223 // The scanner should have, at most, a single result in its cache. If there more results exists 224 // in the cache it means that more than the expected max result size was fetched. 225 assertTrue("The cache contains: " + clientScanner.getCacheSize() + " results", 226 clientScanner.getCacheSize() <= 1); 227 } 228 229 @Test testSmallScan()230 public void testSmallScan() throws Exception { 231 TableName TABLE = TableName.valueOf("testSmallScan"); 232 233 int numRows = 10; 234 byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); 235 236 int numQualifiers = 10; 237 byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); 238 239 Table ht = TEST_UTIL.createTable(TABLE, FAMILY); 240 241 Put put; 242 List<Put> puts = new ArrayList<Put>(); 243 for (int row = 0; row < ROWS.length; row++) { 244 put = new Put(ROWS[row]); 245 for (int qual = 0; qual < QUALIFIERS.length; qual++) { 246 KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE); 247 put.add(kv); 248 } 249 puts.add(put); 250 } 251 ht.put(puts); 252 253 int expectedRows = numRows; 254 int expectedCols = numRows * numQualifiers; 255 256 // Test normal and reversed 257 testSmallScan(ht, true, expectedRows, expectedCols); 258 testSmallScan(ht, false, expectedRows, expectedCols); 259 } 260 261 /** 262 * Run through a variety of test configurations with a small scan 263 * @param table 264 * @param reversed 265 * @param rows 266 * @param columns 267 * @throws Exception 268 */ testSmallScan(Table table, boolean reversed, int rows, int columns)269 public void testSmallScan(Table table, boolean reversed, int rows, int columns) throws Exception { 270 Scan baseScan = new Scan(); 271 baseScan.setReversed(reversed); 272 baseScan.setSmall(true); 273 274 Scan scan = new Scan(baseScan); 275 verifyExpectedCounts(table, scan, rows, columns); 276 277 scan = new Scan(baseScan); 278 scan.setMaxResultSize(1); 279 verifyExpectedCounts(table, scan, rows, columns); 280 281 scan = new Scan(baseScan); 282 scan.setMaxResultSize(1); 283 scan.setCaching(Integer.MAX_VALUE); 284 verifyExpectedCounts(table, scan, rows, columns); 285 } 286 verifyExpectedCounts(Table table, Scan scan, int expectedRowCount, int expectedCellCount)287 private void verifyExpectedCounts(Table table, Scan scan, int expectedRowCount, 288 int expectedCellCount) throws Exception { 289 ResultScanner scanner = table.getScanner(scan); 290 291 int rowCount = 0; 292 int cellCount = 0; 293 Result r = null; 294 while ((r = scanner.next()) != null) { 295 rowCount++; 296 for (Cell c : r.rawCells()) { 297 cellCount++; 298 } 299 } 300 301 assertTrue("Expected row count: " + expectedRowCount + " Actual row count: " + rowCount, 302 expectedRowCount == rowCount); 303 assertTrue("Expected cell count: " + expectedCellCount + " Actual cell count: " + cellCount, 304 expectedCellCount == cellCount); 305 scanner.close(); 306 } 307 308 /** 309 * Test from client side for get with maxResultPerCF set 310 * 311 * @throws Exception 312 */ 313 @Test testGetMaxResults()314 public void testGetMaxResults() throws Exception { 315 byte [] TABLE = Bytes.toBytes("testGetMaxResults"); 316 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); 317 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20); 318 319 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES); 320 321 Get get; 322 Put put; 323 Result result; 324 boolean toLog = true; 325 List<Cell> kvListExp; 326 327 kvListExp = new ArrayList<Cell>(); 328 // Insert one CF for row[0] 329 put = new Put(ROW); 330 for (int i=0; i < 10; i++) { 331 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE); 332 put.add(kv); 333 kvListExp.add(kv); 334 } 335 ht.put(put); 336 337 get = new Get(ROW); 338 result = ht.get(get); 339 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults"); 340 341 get = new Get(ROW); 342 get.setMaxResultsPerColumnFamily(2); 343 result = ht.get(get); 344 kvListExp = new ArrayList<Cell>(); 345 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE)); 346 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE)); 347 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults"); 348 349 // Filters: ColumnRangeFilter 350 get = new Get(ROW); 351 get.setMaxResultsPerColumnFamily(5); 352 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5], 353 true)); 354 result = ht.get(get); 355 kvListExp = new ArrayList<Cell>(); 356 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE)); 357 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE)); 358 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE)); 359 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE)); 360 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF"); 361 362 // Insert two more CF for row[0] 363 // 20 columns for CF2, 10 columns for CF1 364 put = new Put(ROW); 365 for (int i=0; i < QUALIFIERS.length; i++) { 366 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE); 367 put.add(kv); 368 } 369 ht.put(put); 370 371 put = new Put(ROW); 372 for (int i=0; i < 10; i++) { 373 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE); 374 put.add(kv); 375 } 376 ht.put(put); 377 378 get = new Get(ROW); 379 get.setMaxResultsPerColumnFamily(12); 380 get.addFamily(FAMILIES[1]); 381 get.addFamily(FAMILIES[2]); 382 result = ht.get(get); 383 kvListExp = new ArrayList<Cell>(); 384 //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19 385 for (int i=0; i < 10; i++) { 386 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE)); 387 } 388 for (int i=0; i < 2; i++) { 389 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); 390 } 391 for (int i=10; i < 20; i++) { 392 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); 393 } 394 verifyResult(result, kvListExp, toLog, "Testing multiple CFs"); 395 396 // Filters: ColumnRangeFilter and ColumnPrefixFilter 397 get = new Get(ROW); 398 get.setMaxResultsPerColumnFamily(3); 399 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true)); 400 result = ht.get(get); 401 kvListExp = new ArrayList<Cell>(); 402 for (int i=2; i < 5; i++) { 403 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE)); 404 } 405 for (int i=2; i < 5; i++) { 406 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE)); 407 } 408 for (int i=2; i < 5; i++) { 409 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); 410 } 411 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF"); 412 413 get = new Get(ROW); 414 get.setMaxResultsPerColumnFamily(7); 415 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1])); 416 result = ht.get(get); 417 kvListExp = new ArrayList<Cell>(); 418 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE)); 419 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE)); 420 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE)); 421 for (int i=10; i < 16; i++) { 422 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); 423 } 424 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF"); 425 426 } 427 428 /** 429 * Test from client side for scan with maxResultPerCF set 430 * 431 * @throws Exception 432 */ 433 @Test testScanMaxResults()434 public void testScanMaxResults() throws Exception { 435 byte [] TABLE = Bytes.toBytes("testScanLimit"); 436 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2); 437 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); 438 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10); 439 440 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES); 441 442 Put put; 443 Scan scan; 444 Result result; 445 boolean toLog = true; 446 List<Cell> kvListExp, kvListScan; 447 448 kvListExp = new ArrayList<Cell>(); 449 450 for (int r=0; r < ROWS.length; r++) { 451 put = new Put(ROWS[r]); 452 for (int c=0; c < FAMILIES.length; c++) { 453 for (int q=0; q < QUALIFIERS.length; q++) { 454 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE); 455 put.add(kv); 456 if (q < 4) { 457 kvListExp.add(kv); 458 } 459 } 460 } 461 ht.put(put); 462 } 463 464 scan = new Scan(); 465 scan.setMaxResultsPerColumnFamily(4); 466 ResultScanner scanner = ht.getScanner(scan); 467 kvListScan = new ArrayList<Cell>(); 468 while ((result = scanner.next()) != null) { 469 for (Cell kv : result.listCells()) { 470 kvListScan.add(kv); 471 } 472 } 473 result = Result.create(kvListScan); 474 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults"); 475 476 } 477 478 /** 479 * Test from client side for get with rowOffset 480 * 481 * @throws Exception 482 */ 483 @Test testGetRowOffset()484 public void testGetRowOffset() throws Exception { 485 byte [] TABLE = Bytes.toBytes("testGetRowOffset"); 486 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); 487 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20); 488 489 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES); 490 491 Get get; 492 Put put; 493 Result result; 494 boolean toLog = true; 495 List<Cell> kvListExp; 496 497 // Insert one CF for row 498 kvListExp = new ArrayList<Cell>(); 499 put = new Put(ROW); 500 for (int i=0; i < 10; i++) { 501 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE); 502 put.add(kv); 503 // skipping first two kvs 504 if (i < 2) continue; 505 kvListExp.add(kv); 506 } 507 ht.put(put); 508 509 //setting offset to 2 510 get = new Get(ROW); 511 get.setRowOffsetPerColumnFamily(2); 512 result = ht.get(get); 513 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset"); 514 515 //setting offset to 20 516 get = new Get(ROW); 517 get.setRowOffsetPerColumnFamily(20); 518 result = ht.get(get); 519 kvListExp = new ArrayList<Cell>(); 520 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs"); 521 522 //offset + maxResultPerCF 523 get = new Get(ROW); 524 get.setRowOffsetPerColumnFamily(4); 525 get.setMaxResultsPerColumnFamily(5); 526 result = ht.get(get); 527 kvListExp = new ArrayList<Cell>(); 528 for (int i=4; i < 9; i++) { 529 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE)); 530 } 531 verifyResult(result, kvListExp, toLog, 532 "Testing offset + setMaxResultsPerCF"); 533 534 // Filters: ColumnRangeFilter 535 get = new Get(ROW); 536 get.setRowOffsetPerColumnFamily(1); 537 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5], 538 true)); 539 result = ht.get(get); 540 kvListExp = new ArrayList<Cell>(); 541 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE)); 542 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE)); 543 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE)); 544 verifyResult(result, kvListExp, toLog, "Testing offset with CRF"); 545 546 // Insert into two more CFs for row 547 // 10 columns for CF2, 10 columns for CF1 548 for(int j=2; j > 0; j--) { 549 put = new Put(ROW); 550 for (int i=0; i < 10; i++) { 551 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE); 552 put.add(kv); 553 } 554 ht.put(put); 555 } 556 557 get = new Get(ROW); 558 get.setRowOffsetPerColumnFamily(4); 559 get.setMaxResultsPerColumnFamily(2); 560 get.addFamily(FAMILIES[1]); 561 get.addFamily(FAMILIES[2]); 562 result = ht.get(get); 563 kvListExp = new ArrayList<Cell>(); 564 //Exp: CF1:q4, q5, CF2: q4, q5 565 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE)); 566 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE)); 567 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE)); 568 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE)); 569 verifyResult(result, kvListExp, toLog, 570 "Testing offset + multiple CFs + maxResults"); 571 } 572 573 /** 574 * Test from client side for scan while the region is reopened 575 * on the same region server. 576 * 577 * @throws Exception 578 */ 579 @Test testScanOnReopenedRegion()580 public void testScanOnReopenedRegion() throws Exception { 581 TableName TABLE = TableName.valueOf("testScanOnReopenedRegion"); 582 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2); 583 584 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); 585 586 Put put; 587 Scan scan; 588 Result result; 589 ResultScanner scanner; 590 boolean toLog = false; 591 List<Cell> kvListExp; 592 593 // table: row, family, c0:0, c1:1 594 put = new Put(ROW); 595 for (int i=0; i < QUALIFIERS.length; i++) { 596 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE); 597 put.add(kv); 598 } 599 ht.put(put); 600 601 scan = new Scan(ROW); 602 scanner = ht.getScanner(scan); 603 604 HRegionLocation loc = ht.getRegionLocation(ROW); 605 HRegionInfo hri = loc.getRegionInfo(); 606 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); 607 byte[] regionName = hri.getRegionName(); 608 int i = cluster.getServerWith(regionName); 609 HRegionServer rs = cluster.getRegionServer(i); 610 ProtobufUtil.closeRegion(null, 611 rs.getRSRpcServices(), rs.getServerName(), regionName, false); 612 long startTime = EnvironmentEdgeManager.currentTime(); 613 long timeOut = 300000; 614 while (true) { 615 if (rs.getOnlineRegion(regionName) == null) { 616 break; 617 } 618 assertTrue("Timed out in closing the testing region", 619 EnvironmentEdgeManager.currentTime() < startTime + timeOut); 620 Thread.sleep(500); 621 } 622 623 // Now open the region again. 624 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); 625 try { 626 HMaster master = cluster.getMaster(); 627 RegionStates states = master.getAssignmentManager().getRegionStates(); 628 states.regionOffline(hri); 629 states.updateRegionState(hri, State.OPENING); 630 if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) { 631 ZKAssign.createNodeOffline(zkw, hri, loc.getServerName()); 632 } 633 ProtobufUtil.openRegion(null, rs.getRSRpcServices(), rs.getServerName(), hri); 634 startTime = EnvironmentEdgeManager.currentTime(); 635 while (true) { 636 if (rs.getOnlineRegion(regionName) != null) { 637 break; 638 } 639 assertTrue("Timed out in open the testing region", 640 EnvironmentEdgeManager.currentTime() < startTime + timeOut); 641 Thread.sleep(500); 642 } 643 } finally { 644 ZKAssign.deleteNodeFailSilent(zkw, hri); 645 } 646 647 // c0:0, c1:1 648 kvListExp = new ArrayList<Cell>(); 649 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE)); 650 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE)); 651 result = scanner.next(); 652 verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region"); 653 } 654 verifyResult(Result result, List<Cell> expKvList, boolean toLog, String msg)655 static void verifyResult(Result result, List<Cell> expKvList, boolean toLog, 656 String msg) { 657 658 LOG.info(msg); 659 LOG.info("Expected count: " + expKvList.size()); 660 LOG.info("Actual count: " + result.size()); 661 if (expKvList.size() == 0) 662 return; 663 664 int i = 0; 665 for (Cell kv : result.rawCells()) { 666 if (i >= expKvList.size()) { 667 break; // we will check the size later 668 } 669 670 Cell kvExp = expKvList.get(i++); 671 if (toLog) { 672 LOG.info("get kv is: " + kv.toString()); 673 LOG.info("exp kv is: " + kvExp.toString()); 674 } 675 assertTrue("Not equal", kvExp.equals(kv)); 676 } 677 678 assertEquals(expKvList.size(), result.size()); 679 } 680 681 682 } 683