1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.filter; 19 20 import static org.junit.Assert.assertEquals; 21 22 import java.io.IOException; 23 import java.util.ArrayList; 24 import java.util.HashMap; 25 import java.util.HashSet; 26 import java.util.List; 27 import java.util.Map; 28 import java.util.Set; 29 30 import org.apache.hadoop.hbase.Cell; 31 import org.apache.hadoop.hbase.HBaseTestingUtility; 32 import org.apache.hadoop.hbase.HColumnDescriptor; 33 import org.apache.hadoop.hbase.HRegionInfo; 34 import org.apache.hadoop.hbase.HTableDescriptor; 35 import org.apache.hadoop.hbase.KeyValue; 36 import org.apache.hadoop.hbase.KeyValueTestUtil; 37 import org.apache.hadoop.hbase.TableName; 38 import org.apache.hadoop.hbase.client.Durability; 39 import org.apache.hadoop.hbase.client.Put; 40 import org.apache.hadoop.hbase.client.Scan; 41 import org.apache.hadoop.hbase.regionserver.HRegion; 42 import org.apache.hadoop.hbase.regionserver.InternalScanner; 43 import org.apache.hadoop.hbase.testclassification.SmallTests; 44 import org.apache.hadoop.hbase.util.Bytes; 45 import org.junit.Test; 46 import org.junit.experimental.categories.Category; 47 48 @Category(SmallTests.class) 49 public class TestMultipleColumnPrefixFilter { 50 51 private final static HBaseTestingUtility TEST_UTIL = new 52 HBaseTestingUtility(); 53 54 @Test testMultipleColumnPrefixFilter()55 public void testMultipleColumnPrefixFilter() throws IOException { 56 String family = "Family"; 57 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter")); 58 HColumnDescriptor hcd = new HColumnDescriptor(family); 59 hcd.setMaxVersions(3); 60 htd.addFamily(hcd); 61 // HRegionInfo info = new HRegionInfo(htd, null, null, false); 62 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 63 HRegion region = HRegion.createHRegion(info, TEST_UTIL. 64 getDataTestDir(), TEST_UTIL.getConfiguration(), htd); 65 66 List<String> rows = generateRandomWords(100, "row"); 67 List<String> columns = generateRandomWords(10000, "column"); 68 long maxTimestamp = 2; 69 70 List<Cell> kvList = new ArrayList<Cell>(); 71 72 Map<String, List<Cell>> prefixMap = new HashMap<String, 73 List<Cell>>(); 74 75 prefixMap.put("p", new ArrayList<Cell>()); 76 prefixMap.put("q", new ArrayList<Cell>()); 77 prefixMap.put("s", new ArrayList<Cell>()); 78 79 String valueString = "ValueString"; 80 81 for (String row: rows) { 82 Put p = new Put(Bytes.toBytes(row)); 83 p.setDurability(Durability.SKIP_WAL); 84 for (String column: columns) { 85 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 86 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 87 valueString); 88 p.add(kv); 89 kvList.add(kv); 90 for (String s: prefixMap.keySet()) { 91 if (column.startsWith(s)) { 92 prefixMap.get(s).add(kv); 93 } 94 } 95 } 96 } 97 region.put(p); 98 } 99 100 MultipleColumnPrefixFilter filter; 101 Scan scan = new Scan(); 102 scan.setMaxVersions(); 103 byte [][] filter_prefix = new byte [2][]; 104 filter_prefix[0] = new byte [] {'p'}; 105 filter_prefix[1] = new byte [] {'q'}; 106 107 filter = new MultipleColumnPrefixFilter(filter_prefix); 108 scan.setFilter(filter); 109 List<Cell> results = new ArrayList<Cell>(); 110 InternalScanner scanner = region.getScanner(scan); 111 while (scanner.next(results)) 112 ; 113 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); 114 115 HRegion.closeHRegion(region); 116 } 117 118 @Test testMultipleColumnPrefixFilterWithManyFamilies()119 public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException { 120 String family1 = "Family1"; 121 String family2 = "Family2"; 122 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter")); 123 HColumnDescriptor hcd1 = new HColumnDescriptor(family1); 124 hcd1.setMaxVersions(3); 125 htd.addFamily(hcd1); 126 HColumnDescriptor hcd2 = new HColumnDescriptor(family2); 127 hcd2.setMaxVersions(3); 128 htd.addFamily(hcd2); 129 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 130 HRegion region = HRegion.createHRegion(info, TEST_UTIL. 131 getDataTestDir(), TEST_UTIL.getConfiguration(), htd); 132 133 List<String> rows = generateRandomWords(100, "row"); 134 List<String> columns = generateRandomWords(10000, "column"); 135 long maxTimestamp = 3; 136 137 List<Cell> kvList = new ArrayList<Cell>(); 138 139 Map<String, List<Cell>> prefixMap = new HashMap<String, 140 List<Cell>>(); 141 142 prefixMap.put("p", new ArrayList<Cell>()); 143 prefixMap.put("q", new ArrayList<Cell>()); 144 prefixMap.put("s", new ArrayList<Cell>()); 145 146 String valueString = "ValueString"; 147 148 for (String row: rows) { 149 Put p = new Put(Bytes.toBytes(row)); 150 p.setDurability(Durability.SKIP_WAL); 151 for (String column: columns) { 152 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 153 double rand = Math.random(); 154 Cell kv; 155 if (rand < 0.5) 156 kv = KeyValueTestUtil.create(row, family1, column, timestamp, 157 valueString); 158 else 159 kv = KeyValueTestUtil.create(row, family2, column, timestamp, 160 valueString); 161 p.add(kv); 162 kvList.add(kv); 163 for (String s: prefixMap.keySet()) { 164 if (column.startsWith(s)) { 165 prefixMap.get(s).add(kv); 166 } 167 } 168 } 169 } 170 region.put(p); 171 } 172 173 MultipleColumnPrefixFilter filter; 174 Scan scan = new Scan(); 175 scan.setMaxVersions(); 176 byte [][] filter_prefix = new byte [2][]; 177 filter_prefix[0] = new byte [] {'p'}; 178 filter_prefix[1] = new byte [] {'q'}; 179 180 filter = new MultipleColumnPrefixFilter(filter_prefix); 181 scan.setFilter(filter); 182 List<Cell> results = new ArrayList<Cell>(); 183 InternalScanner scanner = region.getScanner(scan); 184 while (scanner.next(results)) 185 ; 186 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); 187 188 HRegion.closeHRegion(region); 189 } 190 191 @Test testMultipleColumnPrefixFilterWithColumnPrefixFilter()192 public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException { 193 String family = "Family"; 194 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter")); 195 htd.addFamily(new HColumnDescriptor(family)); 196 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 197 HRegion region = HRegion.createHRegion(info, TEST_UTIL. 198 getDataTestDir(), TEST_UTIL.getConfiguration(),htd); 199 200 List<String> rows = generateRandomWords(100, "row"); 201 List<String> columns = generateRandomWords(10000, "column"); 202 long maxTimestamp = 2; 203 204 String valueString = "ValueString"; 205 206 for (String row: rows) { 207 Put p = new Put(Bytes.toBytes(row)); 208 p.setDurability(Durability.SKIP_WAL); 209 for (String column: columns) { 210 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 211 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 212 valueString); 213 p.add(kv); 214 } 215 } 216 region.put(p); 217 } 218 219 MultipleColumnPrefixFilter multiplePrefixFilter; 220 Scan scan1 = new Scan(); 221 scan1.setMaxVersions(); 222 byte [][] filter_prefix = new byte [1][]; 223 filter_prefix[0] = new byte [] {'p'}; 224 225 multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix); 226 scan1.setFilter(multiplePrefixFilter); 227 List<Cell> results1 = new ArrayList<Cell>(); 228 InternalScanner scanner1 = region.getScanner(scan1); 229 while (scanner1.next(results1)) 230 ; 231 232 ColumnPrefixFilter singlePrefixFilter; 233 Scan scan2 = new Scan(); 234 scan2.setMaxVersions(); 235 singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p")); 236 237 scan2.setFilter(singlePrefixFilter); 238 List<Cell> results2 = new ArrayList<Cell>(); 239 InternalScanner scanner2 = region.getScanner(scan1); 240 while (scanner2.next(results2)) 241 ; 242 243 assertEquals(results1.size(), results2.size()); 244 245 HRegion.closeHRegion(region); 246 } 247 generateRandomWords(int numberOfWords, String suffix)248 List<String> generateRandomWords(int numberOfWords, String suffix) { 249 Set<String> wordSet = new HashSet<String>(); 250 for (int i = 0; i < numberOfWords; i++) { 251 int lengthOfWords = (int) (Math.random()*2) + 1; 252 char[] wordChar = new char[lengthOfWords]; 253 for (int j = 0; j < wordChar.length; j++) { 254 wordChar[j] = (char) (Math.random() * 26 + 97); 255 } 256 String word; 257 if (suffix == null) { 258 word = new String(wordChar); 259 } else { 260 word = new String(wordChar) + suffix; 261 } 262 wordSet.add(word); 263 } 264 List<String> wordList = new ArrayList<String>(wordSet); 265 return wordList; 266 } 267 268 } 269 270 271