1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase; 19 20 import static org.junit.Assert.assertArrayEquals; 21 import static org.junit.Assert.assertEquals; 22 import static org.junit.Assert.assertFalse; 23 import static org.junit.Assert.assertTrue; 24 import static org.junit.Assert.fail; 25 26 import java.io.IOException; 27 import java.util.ArrayList; 28 import java.util.LinkedHashSet; 29 import java.util.List; 30 import java.util.Set; 31 32 import org.apache.commons.logging.Log; 33 import org.apache.commons.logging.LogFactory; 34 import org.apache.hadoop.hbase.client.ClientScanner; 35 import org.apache.hadoop.hbase.client.Delete; 36 import org.apache.hadoop.hbase.client.Put; 37 import org.apache.hadoop.hbase.client.Result; 38 import org.apache.hadoop.hbase.client.ResultScanner; 39 import org.apache.hadoop.hbase.client.Scan; 40 import org.apache.hadoop.hbase.client.Table; 41 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; 42 import org.apache.hadoop.hbase.filter.ColumnRangeFilter; 43 import org.apache.hadoop.hbase.filter.Filter; 44 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; 45 import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter; 46 import org.apache.hadoop.hbase.filter.RandomRowFilter; 47 import org.apache.hadoop.hbase.testclassification.MediumTests; 48 import org.apache.hadoop.hbase.util.Bytes; 49 import org.apache.hadoop.hbase.util.Pair; 50 import org.junit.AfterClass; 51 import org.junit.BeforeClass; 52 import org.junit.Test; 53 import org.junit.experimental.categories.Category; 54 55 /** 56 * These tests are focused on testing how partial results appear to a client. Partial results are 57 * {@link Result}s that contain only a portion of a row's complete list of cells. Partial results 58 * are formed when the server breaches its maximum result size when trying to service a client's RPC 59 * request. It is the responsibility of the scanner on the client side to recognize when partial 60 * results have been returned and to take action to form the complete results. 61 * <p> 62 * Unless the flag {@link Scan#setAllowPartialResults(boolean)} has been set to true, the caller of 63 * {@link ResultScanner#next()} should never see partial results. 64 */ 65 @Category(MediumTests.class) 66 public class TestPartialResultsFromClientSide { 67 private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class); 68 69 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 70 private final static int MINICLUSTER_SIZE = 5; 71 private static Table TABLE = null; 72 73 /** 74 * Table configuration 75 */ 76 private static TableName TABLE_NAME = TableName.valueOf("testTable"); 77 78 private static int NUM_ROWS = 5; 79 private static byte[] ROW = Bytes.toBytes("testRow"); 80 private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS); 81 82 // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then 83 // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which 84 // breaks the simple generation of expected kv's 85 private static int NUM_FAMILIES = 10; 86 private static byte[] FAMILY = Bytes.toBytes("testFamily"); 87 private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES); 88 89 private static int NUM_QUALIFIERS = 10; 90 private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); 91 private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS); 92 93 private static int VALUE_SIZE = 1024; 94 private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE); 95 96 private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS; 97 98 // Approximation of how large the heap size of cells in our table. Should be accessed through 99 // getCellHeapSize(). 100 private static long CELL_HEAP_SIZE = -1; 101 102 @BeforeClass setUpBeforeClass()103 public static void setUpBeforeClass() throws Exception { 104 TEST_UTIL.startMiniCluster(MINICLUSTER_SIZE); 105 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true); 106 TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE); 107 } 108 createTestTable(TableName name, byte[][] rows, byte[][] families, byte[][] qualifiers, byte[] cellValue)109 static Table createTestTable(TableName name, byte[][] rows, byte[][] families, 110 byte[][] qualifiers, byte[] cellValue) throws IOException { 111 Table ht = TEST_UTIL.createTable(name, families); 112 List<Put> puts = createPuts(rows, families, qualifiers, cellValue); 113 ht.put(puts); 114 115 return ht; 116 } 117 118 @AfterClass tearDownAfterClass()119 public static void tearDownAfterClass() throws Exception { 120 TEST_UTIL.shutdownMiniCluster(); 121 } 122 123 /** 124 * Ensure that the expected key values appear in a result returned from a scanner that is 125 * combining partial results into complete results 126 * @throws Exception 127 */ 128 @Test testExpectedValuesOfPartialResults()129 public void testExpectedValuesOfPartialResults() throws Exception { 130 testExpectedValuesOfPartialResults(false); 131 testExpectedValuesOfPartialResults(true); 132 } 133 testExpectedValuesOfPartialResults(boolean reversed)134 public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception { 135 Scan partialScan = new Scan(); 136 partialScan.setMaxVersions(); 137 // Max result size of 1 ensures that each RPC request will return a single cell. The scanner 138 // will need to reconstruct the results into a complete result before returning to the caller 139 partialScan.setMaxResultSize(1); 140 partialScan.setReversed(reversed); 141 ResultScanner partialScanner = TABLE.getScanner(partialScan); 142 143 final int startRow = reversed ? ROWS.length - 1 : 0; 144 final int endRow = reversed ? -1 : ROWS.length; 145 final int loopDelta = reversed ? -1 : 1; 146 String message; 147 148 for (int row = startRow; row != endRow; row = row + loopDelta) { 149 message = "Ensuring the expected keyValues are present for row " + row; 150 List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE); 151 Result result = partialScanner.next(); 152 assertFalse(result.isPartial()); 153 verifyResult(result, expectedKeyValues, message); 154 } 155 156 partialScanner.close(); 157 } 158 159 /** 160 * Ensure that we only see Results marked as partial when the allowPartial flag is set 161 * @throws Exception 162 */ 163 @Test testAllowPartialResults()164 public void testAllowPartialResults() throws Exception { 165 Scan scan = new Scan(); 166 scan.setAllowPartialResults(true); 167 scan.setMaxResultSize(1); 168 ResultScanner scanner = TABLE.getScanner(scan); 169 Result result = scanner.next(); 170 171 assertTrue(result != null); 172 assertTrue(result.isPartial()); 173 assertTrue(result.rawCells() != null); 174 assertTrue(result.rawCells().length == 1); 175 176 scanner.close(); 177 178 scan.setAllowPartialResults(false); 179 scanner = TABLE.getScanner(scan); 180 result = scanner.next(); 181 182 assertTrue(result != null); 183 assertTrue(!result.isPartial()); 184 assertTrue(result.rawCells() != null); 185 assertTrue(result.rawCells().length == NUM_COLS); 186 187 scanner.close(); 188 } 189 190 /** 191 * Ensure that the results returned from a scanner that retrieves all results in a single RPC call 192 * matches the results that are returned from a scanner that must incrementally combine partial 193 * results into complete results. A variety of scan configurations can be tested 194 * @throws Exception 195 */ 196 @Test testEquivalenceOfScanResults()197 public void testEquivalenceOfScanResults() throws Exception { 198 Scan oneShotScan = new Scan(); 199 oneShotScan.setMaxResultSize(Long.MAX_VALUE); 200 201 Scan partialScan = new Scan(oneShotScan); 202 partialScan.setMaxResultSize(1); 203 204 testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan); 205 } 206 testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2)207 public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception { 208 ResultScanner scanner1 = table.getScanner(scan1); 209 ResultScanner scanner2 = table.getScanner(scan2); 210 211 Result r1 = null; 212 Result r2 = null; 213 int count = 0; 214 215 while ((r1 = scanner1.next()) != null) { 216 r2 = scanner2.next(); 217 218 assertTrue(r2 != null); 219 compareResults(r1, r2, "Comparing result #" + count); 220 count++; 221 } 222 223 r2 = scanner2.next(); 224 assertTrue("r2: " + r2 + " Should be null", r2 == null); 225 226 scanner1.close(); 227 scanner2.close(); 228 } 229 230 /** 231 * Order of cells in partial results matches the ordering of cells from complete results 232 * @throws Exception 233 */ 234 @Test testOrderingOfCellsInPartialResults()235 public void testOrderingOfCellsInPartialResults() throws Exception { 236 Scan scan = new Scan(); 237 238 for (int col = 1; col <= NUM_COLS; col++) { 239 scan.setMaxResultSize(getResultSizeForNumberOfCells(col)); 240 testOrderingOfCellsInPartialResults(scan); 241 242 // Test again with a reversed scanner 243 scan.setReversed(true); 244 testOrderingOfCellsInPartialResults(scan); 245 } 246 } 247 testOrderingOfCellsInPartialResults(final Scan basePartialScan)248 public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception { 249 // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true 250 // the results will NOT be reconstructed and instead the caller will see the partial results 251 // returned by the server 252 Scan partialScan = new Scan(basePartialScan); 253 partialScan.setAllowPartialResults(true); 254 ResultScanner partialScanner = TABLE.getScanner(partialScan); 255 256 // Scan that retrieves all table results in single RPC request 257 Scan oneShotScan = new Scan(basePartialScan); 258 oneShotScan.setMaxResultSize(Long.MAX_VALUE); 259 oneShotScan.setCaching(ROWS.length); 260 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); 261 262 Result oneShotResult = oneShotScanner.next(); 263 Result partialResult = null; 264 int iterationCount = 0; 265 266 while (oneShotResult != null && oneShotResult.rawCells() != null) { 267 List<Cell> aggregatePartialCells = new ArrayList<Cell>(); 268 do { 269 partialResult = partialScanner.next(); 270 assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null); 271 assertTrue("Partial cells are null. iteration: " + iterationCount, 272 partialResult.rawCells() != null); 273 274 for (Cell c : partialResult.rawCells()) { 275 aggregatePartialCells.add(c); 276 } 277 } while (partialResult.isPartial()); 278 279 assertTrue("Number of cells differs. iteration: " + iterationCount, 280 oneShotResult.rawCells().length == aggregatePartialCells.size()); 281 final Cell[] oneShotCells = oneShotResult.rawCells(); 282 for (int cell = 0; cell < oneShotCells.length; cell++) { 283 Cell oneShotCell = oneShotCells[cell]; 284 Cell partialCell = aggregatePartialCells.get(cell); 285 286 assertTrue("One shot cell was null", oneShotCell != null); 287 assertTrue("Partial cell was null", partialCell != null); 288 assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell, 289 oneShotCell.equals(partialCell)); 290 } 291 292 oneShotResult = oneShotScanner.next(); 293 iterationCount++; 294 } 295 296 assertTrue(partialScanner.next() == null); 297 298 partialScanner.close(); 299 oneShotScanner.close(); 300 } 301 302 /** 303 * Setting the max result size allows us to control how many cells we expect to see on each call 304 * to next on the scanner. Test a variety of different sizes for correctness 305 * @throws Exception 306 */ 307 @Test testExpectedNumberOfCellsPerPartialResult()308 public void testExpectedNumberOfCellsPerPartialResult() throws Exception { 309 Scan scan = new Scan(); 310 testExpectedNumberOfCellsPerPartialResult(scan); 311 312 scan.setReversed(true); 313 testExpectedNumberOfCellsPerPartialResult(scan); 314 } 315 testExpectedNumberOfCellsPerPartialResult(Scan baseScan)316 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception { 317 for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) { 318 testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells); 319 } 320 } 321 testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells)322 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells) 323 throws Exception { 324 325 if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells); 326 327 // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back 328 // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less 329 // than it in cases where expectedNumberOfCells is not an exact multiple of the number of 330 // columns in the table. 331 Scan scan = new Scan(baseScan); 332 scan.setAllowPartialResults(true); 333 scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells)); 334 335 ResultScanner scanner = TABLE.getScanner(scan); 336 Result result = null; 337 byte[] prevRow = null; 338 while ((result = scanner.next()) != null) { 339 assertTrue(result.rawCells() != null); 340 341 // Cases when cell count won't equal expectedNumberOfCells: 342 // 1. Returned result is the final result needed to form the complete result for that row 343 // 2. It is the first result we have seen for that row and thus may have been fetched as 344 // the last group of cells that fit inside the maxResultSize 345 assertTrue( 346 "Result's cell count differed from expected number. result: " + result, 347 result.rawCells().length == expectedNumberOfCells || !result.isPartial() 348 || !Bytes.equals(prevRow, result.getRow())); 349 prevRow = result.getRow(); 350 } 351 352 scanner.close(); 353 } 354 355 /** 356 * @return The approximate heap size of a cell in the test table. All cells should have 357 * approximately the same heap size, so the value is cached to avoid repeating the 358 * calculation 359 * @throws Exception 360 */ getCellHeapSize()361 private long getCellHeapSize() throws Exception { 362 if (CELL_HEAP_SIZE == -1) { 363 // Do a partial scan that will return a single result with a single cell 364 Scan scan = new Scan(); 365 scan.setMaxResultSize(1); 366 scan.setAllowPartialResults(true); 367 ResultScanner scanner = TABLE.getScanner(scan); 368 369 Result result = scanner.next(); 370 371 assertTrue(result != null); 372 assertTrue(result.rawCells() != null); 373 assertTrue(result.rawCells().length == 1); 374 375 CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]); 376 if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE); 377 scanner.close(); 378 } 379 380 return CELL_HEAP_SIZE; 381 } 382 383 /** 384 * @param numberOfCells 385 * @return the result size that should be used in {@link Scan#setMaxResultSize(long)} if you want 386 * the server to return exactly numberOfCells cells 387 * @throws Exception 388 */ getResultSizeForNumberOfCells(int numberOfCells)389 private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception { 390 return getCellHeapSize() * numberOfCells; 391 } 392 393 /** 394 * Test various combinations of batching and partial results for correctness 395 */ 396 @Test testPartialResultsAndBatch()397 public void testPartialResultsAndBatch() throws Exception { 398 for (int batch = 1; batch <= NUM_COLS / 4; batch++) { 399 for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) { 400 testPartialResultsAndBatch(batch, cellsPerPartial); 401 } 402 } 403 } 404 testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)405 public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult) 406 throws Exception { 407 if (LOG.isInfoEnabled()) { 408 LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult); 409 } 410 411 Scan scan = new Scan(); 412 scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult)); 413 scan.setBatch(batch); 414 ResultScanner scanner = TABLE.getScanner(scan); 415 Result result = scanner.next(); 416 int repCount = 0; 417 418 while ((result = scanner.next()) != null) { 419 assertTrue(result.rawCells() != null); 420 421 if (result.isPartial()) { 422 final String error = 423 "Cells:" + result.rawCells().length + " Batch size:" + batch 424 + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount; 425 assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult)); 426 } else { 427 assertTrue(result.rawCells().length <= batch); 428 } 429 repCount++; 430 } 431 432 scanner.close(); 433 } 434 435 /** 436 * Test the method {@link Result#createCompleteResult(List)} 437 * @throws Exception 438 */ 439 @Test testPartialResultsReassembly()440 public void testPartialResultsReassembly() throws Exception { 441 Scan scan = new Scan(); 442 testPartialResultsReassembly(scan); 443 scan.setReversed(true); 444 testPartialResultsReassembly(scan); 445 } 446 testPartialResultsReassembly(Scan scanBase)447 public void testPartialResultsReassembly(Scan scanBase) throws Exception { 448 Scan partialScan = new Scan(scanBase); 449 partialScan.setMaxResultSize(1); 450 partialScan.setAllowPartialResults(true); 451 ResultScanner partialScanner = TABLE.getScanner(partialScan); 452 453 Scan oneShotScan = new Scan(scanBase); 454 oneShotScan.setMaxResultSize(Long.MAX_VALUE); 455 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); 456 457 ArrayList<Result> partials = new ArrayList<>(); 458 for (int i = 0; i < NUM_ROWS; i++) { 459 Result partialResult = null; 460 Result completeResult = null; 461 Result oneShotResult = null; 462 partials.clear(); 463 464 do { 465 partialResult = partialScanner.next(); 466 partials.add(partialResult); 467 } while (partialResult != null && partialResult.isPartial()); 468 469 completeResult = Result.createCompleteResult(partials); 470 oneShotResult = oneShotScanner.next(); 471 472 compareResults(completeResult, oneShotResult, null); 473 } 474 475 assertTrue(oneShotScanner.next() == null); 476 assertTrue(partialScanner.next() == null); 477 478 oneShotScanner.close(); 479 partialScanner.close(); 480 } 481 482 /** 483 * When reconstructing the complete result from its partials we ensure that the row of each 484 * partial result is the same. If one of the rows differs, an exception is thrown. 485 */ 486 @Test testExceptionThrownOnMismatchedPartialResults()487 public void testExceptionThrownOnMismatchedPartialResults() throws IOException { 488 assertTrue(NUM_ROWS >= 2); 489 490 ArrayList<Result> partials = new ArrayList<>(); 491 Scan scan = new Scan(); 492 scan.setMaxResultSize(Long.MAX_VALUE); 493 ResultScanner scanner = TABLE.getScanner(scan); 494 Result r1 = scanner.next(); 495 partials.add(r1); 496 Result r2 = scanner.next(); 497 partials.add(r2); 498 499 assertFalse(Bytes.equals(r1.getRow(), r2.getRow())); 500 501 try { 502 Result.createCompleteResult(partials); 503 fail("r1 and r2 are from different rows. It should not be possible to combine them into" 504 + " a single result"); 505 } catch (IOException e) { 506 } 507 508 scanner.close(); 509 } 510 511 /** 512 * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is 513 * true, the scanner should not return partial results. The scanner cannot return partial results 514 * because the entire row needs to be read for the include/exclude decision to be made 515 */ 516 @Test testNoPartialResultsWhenRowFilterPresent()517 public void testNoPartialResultsWhenRowFilterPresent() throws Exception { 518 Scan scan = new Scan(); 519 scan.setMaxResultSize(1); 520 scan.setAllowPartialResults(true); 521 // If a filter hasFilter() is true then partial results should not be returned else filter 522 // application server side would break. 523 scan.setFilter(new RandomRowFilter(1.0f)); 524 ResultScanner scanner = TABLE.getScanner(scan); 525 526 Result r = null; 527 while ((r = scanner.next()) != null) { 528 assertFalse(r.isPartial()); 529 } 530 531 scanner.close(); 532 } 533 534 /** 535 * Examine the interaction between the maxResultSize and caching. If the caching limit is reached 536 * before the maxResultSize limit, we should not see partial results. On the other hand, if the 537 * maxResultSize limit is reached before the caching limit, it is likely that partial results will 538 * be seen. 539 * @throws Exception 540 */ 541 @Test testPartialResultsAndCaching()542 public void testPartialResultsAndCaching() throws Exception { 543 for (int caching = 1; caching <= NUM_ROWS; caching++) { 544 for (int maxResultRows = 0; maxResultRows <= NUM_ROWS; maxResultRows++) { 545 testPartialResultsAndCaching(maxResultRows, caching); 546 } 547 } 548 } 549 550 /** 551 * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize 552 * @param cachingRowLimit The row limit that will be enforced through caching 553 * @throws Exception 554 */ testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)555 public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit) 556 throws Exception { 557 Scan scan = new Scan(); 558 scan.setAllowPartialResults(true); 559 560 // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to 561 // ensure that the result size we specify is not an exact multiple of the number of cells 562 // in a row. This ensures that partial results will be returned when the result size limit 563 // is reached before the caching limit. 564 int cellOffset = NUM_COLS / 3; 565 long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset); 566 scan.setMaxResultSize(maxResultSize); 567 scan.setCaching(cachingRowLimit); 568 569 ResultScanner scanner = TABLE.getScanner(scan); 570 ClientScanner clientScanner = (ClientScanner) scanner; 571 Result r = null; 572 573 // Approximate the number of rows we expect will fit into the specified max rsult size. If this 574 // approximation is less than caching, then we expect that the max result size limit will be 575 // hit before the caching limit and thus partial results may be seen 576 boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit; 577 while ((r = clientScanner.next()) != null) { 578 assertTrue(!r.isPartial() || expectToSeePartialResults); 579 } 580 581 scanner.close(); 582 } 583 584 /** 585 * Small scans should not return partial results because it would prevent small scans from 586 * retrieving all of the necessary results in a single RPC request which is what makese small 587 * scans useful. Thus, ensure that even when {@link Scan#getAllowPartialResults()} is true, small 588 * scans do not return partial results 589 * @throws Exception 590 */ 591 @Test 592 public void testSmallScansDoNotAllowPartials() throws Exception { 593 Scan scan = new Scan(); 594 testSmallScansDoNotAllowPartials(scan); 595 scan.setReversed(true); 596 testSmallScansDoNotAllowPartials(scan); 597 } 598 599 public void testSmallScansDoNotAllowPartials(Scan baseScan) throws Exception { 600 Scan scan = new Scan(baseScan); 601 scan.setAllowPartialResults(true); 602 scan.setSmall(true); 603 scan.setMaxResultSize(1); 604 605 ResultScanner scanner = TABLE.getScanner(scan); 606 Result r = null; 607 608 while ((r = scanner.next()) != null) { 609 assertFalse(r.isPartial()); 610 } 611 612 scanner.close(); 613 } 614 615 /** 616 * Make puts to put the input value into each combination of row, family, and qualifier 617 * @param rows 618 * @param families 619 * @param qualifiers 620 * @param value 621 * @return 622 * @throws IOException 623 */ 624 static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers, 625 byte[] value) throws IOException { 626 Put put; 627 ArrayList<Put> puts = new ArrayList<>(); 628 629 for (int row = 0; row < rows.length; row++) { 630 put = new Put(rows[row]); 631 for (int fam = 0; fam < families.length; fam++) { 632 for (int qual = 0; qual < qualifiers.length; qual++) { 633 KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value); 634 put.add(kv); 635 } 636 } 637 puts.add(put); 638 } 639 640 return puts; 641 } 642 643 /** 644 * Make key values to represent each possible combination of family and qualifier in the specified 645 * row. 646 * @param row 647 * @param families 648 * @param qualifiers 649 * @param value 650 * @return 651 */ 652 static ArrayList<Cell> createKeyValuesForRow(byte[] row, byte[][] families, byte[][] qualifiers, 653 byte[] value) { 654 ArrayList<Cell> outList = new ArrayList<>(); 655 for (int fam = 0; fam < families.length; fam++) { 656 for (int qual = 0; qual < qualifiers.length; qual++) { 657 outList.add(new KeyValue(row, families[fam], qualifiers[qual], qual, value)); 658 } 659 } 660 return outList; 661 } 662 663 /** 664 * Verifies that result contains all the key values within expKvList. Fails the test otherwise 665 * @param result 666 * @param expKvList 667 * @param msg 668 */ 669 static void verifyResult(Result result, List<Cell> expKvList, String msg) { 670 if (LOG.isInfoEnabled()) { 671 LOG.info(msg); 672 LOG.info("Expected count: " + expKvList.size()); 673 LOG.info("Actual count: " + result.size()); 674 } 675 676 if (expKvList.size() == 0) return; 677 678 int i = 0; 679 for (Cell kv : result.rawCells()) { 680 if (i >= expKvList.size()) { 681 break; // we will check the size later 682 } 683 684 Cell kvExp = expKvList.get(i++); 685 assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(), 686 kvExp.equals(kv)); 687 } 688 689 assertEquals(expKvList.size(), result.size()); 690 } 691 692 /** 693 * Compares two results and fails the test if the results are different 694 * @param r1 695 * @param r2 696 * @param message 697 */ 698 static void compareResults(Result r1, Result r2, final String message) { 699 if (LOG.isInfoEnabled()) { 700 if (message != null) LOG.info(message); 701 LOG.info("r1: " + r1); 702 LOG.info("r2: " + r2); 703 } 704 705 final String failureMessage = "Results r1:" + r1 + " \nr2:" + r2 + " are not equivalent"; 706 if (r1 == null && r2 == null) fail(failureMessage); 707 else if (r1 == null || r2 == null) fail(failureMessage); 708 709 try { 710 Result.compareResults(r1, r2); 711 } catch (Exception e) { 712 fail(failureMessage); 713 } 714 } 715 716 @Test 717 public void testReadPointAndPartialResults() throws Exception { 718 TableName testName = TableName.valueOf("testReadPointAndPartialResults"); 719 int numRows = 5; 720 int numFamilies = 5; 721 int numQualifiers = 5; 722 byte[][] rows = HTestConst.makeNAscii(Bytes.toBytes("testRow"), numRows); 723 byte[][] families = HTestConst.makeNAscii(Bytes.toBytes("testFamily"), numFamilies); 724 byte[][] qualifiers = HTestConst.makeNAscii(Bytes.toBytes("testQualifier"), numQualifiers); 725 byte[] value = Bytes.createMaxByteArray(100); 726 727 Table tmpTable = createTestTable(testName, rows, families, qualifiers, value); 728 729 Scan scan = new Scan(); 730 scan.setMaxResultSize(1); 731 scan.setAllowPartialResults(true); 732 733 // Open scanner before deletes 734 ResultScanner scanner = tmpTable.getScanner(scan); 735 736 Delete delete1 = new Delete(rows[0]); 737 delete1.addColumn(families[0], qualifiers[0], 0); 738 tmpTable.delete(delete1); 739 740 Delete delete2 = new Delete(rows[1]); 741 delete2.addColumn(families[1], qualifiers[1], 1); 742 tmpTable.delete(delete2); 743 744 // Should see all cells because scanner was opened prior to deletes 745 int scannerCount = countCellsFromScanner(scanner); 746 int expectedCount = numRows * numFamilies * numQualifiers; 747 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount, 748 scannerCount == expectedCount); 749 750 // Minus 2 for the two cells that were deleted 751 scanner = tmpTable.getScanner(scan); 752 scannerCount = countCellsFromScanner(scanner); 753 expectedCount = numRows * numFamilies * numQualifiers - 2; 754 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount, 755 scannerCount == expectedCount); 756 757 scanner = tmpTable.getScanner(scan); 758 // Put in 2 new rows. The timestamps differ from the deleted rows 759 Put put1 = new Put(rows[0]); 760 put1.add(new KeyValue(rows[0], families[0], qualifiers[0], 1, value)); 761 tmpTable.put(put1); 762 763 Put put2 = new Put(rows[1]); 764 put2.add(new KeyValue(rows[1], families[1], qualifiers[1], 2, value)); 765 tmpTable.put(put2); 766 767 // Scanner opened prior to puts. Cell count shouldn't have changed 768 scannerCount = countCellsFromScanner(scanner); 769 expectedCount = numRows * numFamilies * numQualifiers - 2; 770 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount, 771 scannerCount == expectedCount); 772 773 // Now the scanner should see the cells that were added by puts 774 scanner = tmpTable.getScanner(scan); 775 scannerCount = countCellsFromScanner(scanner); 776 expectedCount = numRows * numFamilies * numQualifiers; 777 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount, 778 scannerCount == expectedCount); 779 780 TEST_UTIL.deleteTable(testName); 781 } 782 783 /** 784 * Exhausts the scanner by calling next repetitively. Once completely exhausted, close scanner and 785 * return total cell count 786 * @param scanner 787 * @return 788 * @throws Exception 789 */ 790 private int countCellsFromScanner(ResultScanner scanner) throws Exception { 791 Result result = null; 792 int numCells = 0; 793 while ((result = scanner.next()) != null) { 794 numCells += result.rawCells().length; 795 } 796 797 scanner.close(); 798 return numCells; 799 } 800 801 /** 802 * Test partial Result re-assembly in the presence of different filters. The Results from the 803 * partial scanner should match the Results returned from a scanner that receives all of the 804 * results in one RPC to the server. The partial scanner is tested with a variety of different 805 * result sizes (all of which are less than the size necessary to fetch an entire row) 806 * @throws Exception 807 */ 808 @Test 809 public void testPartialResultsWithColumnFilter() throws Exception { 810 testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter()); 811 testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5"))); 812 testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true, 813 Bytes.toBytes("testQualifier7"), true)); 814 815 Set<byte[]> qualifiers = new LinkedHashSet<>(); 816 qualifiers.add(Bytes.toBytes("testQualifier5")); 817 testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers)); 818 } 819 820 public void testPartialResultsWithColumnFilter(Filter filter) throws Exception { 821 assertTrue(!filter.hasFilterRow()); 822 823 Scan partialScan = new Scan(); 824 partialScan.setFilter(filter); 825 826 Scan oneshotScan = new Scan(); 827 oneshotScan.setFilter(filter); 828 oneshotScan.setMaxResultSize(Long.MAX_VALUE); 829 830 for (int i = 1; i <= NUM_COLS; i++) { 831 partialScan.setMaxResultSize(getResultSizeForNumberOfCells(i)); 832 testEquivalenceOfScanResults(TABLE, partialScan, oneshotScan); 833 } 834 } 835 836 private void moveRegion(Table table, int index) throws IOException{ 837 List<Pair<HRegionInfo, ServerName>> regions = MetaTableAccessor 838 .getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(), 839 table.getName()); 840 assertEquals(1, regions.size()); 841 HRegionInfo regionInfo = regions.get(0).getFirst(); 842 ServerName name = TEST_UTIL.getHBaseCluster().getRegionServer(index).getServerName(); 843 TEST_UTIL.getHBaseAdmin().move(regionInfo.getEncodedNameAsBytes(), 844 Bytes.toBytes(name.getServerName())); 845 } 846 847 private void assertCell(Cell cell, byte[] row, byte[] cf, byte[] cq) { 848 assertArrayEquals(row, 849 Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); 850 assertArrayEquals(cf, 851 Bytes.copy(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); 852 assertArrayEquals(cq, 853 Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); 854 } 855 856 @Test 857 public void testPartialResultWhenRegionMove() throws IOException { 858 Table table=createTestTable(TableName.valueOf("testPartialResultWhenRegionMove"), 859 ROWS, FAMILIES, QUALIFIERS, VALUE); 860 861 moveRegion(table, 1); 862 863 Scan scan = new Scan(); 864 scan.setMaxResultSize(1); 865 scan.setAllowPartialResults(true); 866 ResultScanner scanner = table.getScanner(scan); 867 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) { 868 scanner.next(); 869 } 870 Result result1 = scanner.next(); 871 assertEquals(1, result1.rawCells().length); 872 Cell c1 = result1.rawCells()[0]; 873 assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]); 874 assertFalse(result1.isPartial()); 875 876 moveRegion(table, 2); 877 878 Result result2 = scanner.next(); 879 assertEquals(1, result2.rawCells().length); 880 Cell c2 = result2.rawCells()[0]; 881 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]); 882 assertTrue(result2.isPartial()); 883 884 moveRegion(table, 3); 885 886 Result result3 = scanner.next(); 887 assertEquals(1, result3.rawCells().length); 888 Cell c3 = result3.rawCells()[0]; 889 assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]); 890 assertTrue(result3.isPartial()); 891 892 } 893 894 @Test 895 public void testReversedPartialResultWhenRegionMove() throws IOException { 896 Table table=createTestTable(TableName.valueOf("testReversedPartialResultWhenRegionMove"), 897 ROWS, FAMILIES, QUALIFIERS, VALUE); 898 899 moveRegion(table, 1); 900 901 Scan scan = new Scan(); 902 scan.setMaxResultSize(1); 903 scan.setAllowPartialResults(true); 904 scan.setReversed(true); 905 ResultScanner scanner = table.getScanner(scan); 906 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS-1; i++) { 907 scanner.next(); 908 } 909 Result result1 = scanner.next(); 910 assertEquals(1, result1.rawCells().length); 911 Cell c1 = result1.rawCells()[0]; 912 assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]); 913 assertFalse(result1.isPartial()); 914 915 moveRegion(table, 2); 916 917 Result result2 = scanner.next(); 918 assertEquals(1, result2.rawCells().length); 919 Cell c2 = result2.rawCells()[0]; 920 assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]); 921 assertTrue(result2.isPartial()); 922 923 moveRegion(table, 3); 924 925 Result result3 = scanner.next(); 926 assertEquals(1, result3.rawCells().length); 927 Cell c3 = result3.rawCells()[0]; 928 assertCell(c3, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[1]); 929 assertTrue(result3.isPartial()); 930 931 } 932 933 @Test 934 public void testCompleteResultWhenRegionMove() throws IOException { 935 Table table=createTestTable(TableName.valueOf("testCompleteResultWhenRegionMove"), 936 ROWS, FAMILIES, QUALIFIERS, VALUE); 937 938 moveRegion(table, 1); 939 940 Scan scan = new Scan(); 941 scan.setMaxResultSize(1); 942 scan.setCaching(1); 943 ResultScanner scanner = table.getScanner(scan); 944 945 Result result1 = scanner.next(); 946 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result1.rawCells().length); 947 Cell c1 = result1.rawCells()[0]; 948 assertCell(c1, ROWS[0], FAMILIES[0], QUALIFIERS[0]); 949 assertFalse(result1.isPartial()); 950 951 moveRegion(table, 2); 952 953 Result result2 = scanner.next(); 954 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result2.rawCells().length); 955 Cell c2 = result2.rawCells()[0]; 956 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]); 957 assertFalse(result2.isPartial()); 958 959 moveRegion(table, 3); 960 961 Result result3 = scanner.next(); 962 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result3.rawCells().length); 963 Cell c3 = result3.rawCells()[0]; 964 assertCell(c3, ROWS[2], FAMILIES[0], QUALIFIERS[0]); 965 assertFalse(result3.isPartial()); 966 967 } 968 969 @Test 970 public void testReversedCompleteResultWhenRegionMove() throws IOException { 971 Table table=createTestTable(TableName.valueOf("testReversedCompleteResultWhenRegionMove"), 972 ROWS, FAMILIES, QUALIFIERS, VALUE); 973 974 moveRegion(table, 1); 975 976 Scan scan = new Scan(); 977 scan.setMaxResultSize(1); 978 scan.setCaching(1); 979 scan.setReversed(true); 980 ResultScanner scanner = table.getScanner(scan); 981 982 Result result1 = scanner.next(); 983 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result1.rawCells().length); 984 Cell c1 = result1.rawCells()[0]; 985 assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[0], QUALIFIERS[0]); 986 assertFalse(result1.isPartial()); 987 988 moveRegion(table, 2); 989 990 Result result2 = scanner.next(); 991 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result2.rawCells().length); 992 Cell c2 = result2.rawCells()[0]; 993 assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]); 994 assertFalse(result2.isPartial()); 995 996 moveRegion(table, 3); 997 998 Result result3 = scanner.next(); 999 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result3.rawCells().length); 1000 Cell c3 = result3.rawCells()[0]; 1001 assertCell(c3, ROWS[NUM_ROWS-3], FAMILIES[0], QUALIFIERS[0]); 1002 assertFalse(result3.isPartial()); 1003 1004 } 1005 1006 @Test 1007 public void testBatchingResultWhenRegionMove() throws IOException { 1008 Table table = 1009 createTestTable(TableName.valueOf("testBatchingResultWhenRegionMove"), ROWS, FAMILIES, 1010 QUALIFIERS, VALUE); 1011 1012 moveRegion(table, 1); 1013 1014 Scan scan = new Scan(); 1015 scan.setCaching(1); 1016 scan.setBatch(1); 1017 1018 ResultScanner scanner = table.getScanner(scan); 1019 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) { 1020 scanner.next(); 1021 } 1022 Result result1 = scanner.next(); 1023 assertEquals(1, result1.rawCells().length); 1024 Cell c1 = result1.rawCells()[0]; 1025 assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]); 1026 1027 moveRegion(table, 2); 1028 1029 Result result2 = scanner.next(); 1030 assertEquals(1, result2.rawCells().length); 1031 Cell c2 = result2.rawCells()[0]; 1032 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]); 1033 1034 moveRegion(table, 3); 1035 1036 Result result3 = scanner.next(); 1037 assertEquals(1, result3.rawCells().length); 1038 Cell c3 = result3.rawCells()[0]; 1039 assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]); 1040 } 1041 1042 1043 }