1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 package org.apache.hadoop.hbase.filter;
19 
20 import static org.junit.Assert.assertEquals;
21 
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Set;
29 
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValueTestUtil;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.Durability;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.regionserver.HRegion;
42 import org.apache.hadoop.hbase.regionserver.InternalScanner;
43 import org.apache.hadoop.hbase.testclassification.SmallTests;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47 
48 @Category(SmallTests.class)
49 public class TestMultipleColumnPrefixFilter {
50 
51   private final static HBaseTestingUtility TEST_UTIL = new
52       HBaseTestingUtility();
53 
54   @Test
testMultipleColumnPrefixFilter()55   public void testMultipleColumnPrefixFilter() throws IOException {
56     String family = "Family";
57     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
58     HColumnDescriptor hcd = new HColumnDescriptor(family);
59     hcd.setMaxVersions(3);
60     htd.addFamily(hcd);
61     // HRegionInfo info = new HRegionInfo(htd, null, null, false);
62     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
63     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
64       getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
65 
66     List<String> rows = generateRandomWords(100, "row");
67     List<String> columns = generateRandomWords(10000, "column");
68     long maxTimestamp = 2;
69 
70     List<Cell> kvList = new ArrayList<Cell>();
71 
72     Map<String, List<Cell>> prefixMap = new HashMap<String,
73         List<Cell>>();
74 
75     prefixMap.put("p", new ArrayList<Cell>());
76     prefixMap.put("q", new ArrayList<Cell>());
77     prefixMap.put("s", new ArrayList<Cell>());
78 
79     String valueString = "ValueString";
80 
81     for (String row: rows) {
82       Put p = new Put(Bytes.toBytes(row));
83       p.setDurability(Durability.SKIP_WAL);
84       for (String column: columns) {
85         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
86           KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
87               valueString);
88           p.add(kv);
89           kvList.add(kv);
90           for (String s: prefixMap.keySet()) {
91             if (column.startsWith(s)) {
92               prefixMap.get(s).add(kv);
93             }
94           }
95         }
96       }
97       region.put(p);
98     }
99 
100     MultipleColumnPrefixFilter filter;
101     Scan scan = new Scan();
102     scan.setMaxVersions();
103     byte [][] filter_prefix = new byte [2][];
104     filter_prefix[0] = new byte [] {'p'};
105     filter_prefix[1] = new byte [] {'q'};
106 
107     filter = new MultipleColumnPrefixFilter(filter_prefix);
108     scan.setFilter(filter);
109     List<Cell> results = new ArrayList<Cell>();
110     InternalScanner scanner = region.getScanner(scan);
111     while (scanner.next(results))
112       ;
113     assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
114 
115     HRegion.closeHRegion(region);
116   }
117 
118   @Test
testMultipleColumnPrefixFilterWithManyFamilies()119   public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
120     String family1 = "Family1";
121     String family2 = "Family2";
122     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
123     HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
124     hcd1.setMaxVersions(3);
125     htd.addFamily(hcd1);
126     HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
127     hcd2.setMaxVersions(3);
128     htd.addFamily(hcd2);
129     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
130     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
131       getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
132 
133     List<String> rows = generateRandomWords(100, "row");
134     List<String> columns = generateRandomWords(10000, "column");
135     long maxTimestamp = 3;
136 
137     List<Cell> kvList = new ArrayList<Cell>();
138 
139     Map<String, List<Cell>> prefixMap = new HashMap<String,
140         List<Cell>>();
141 
142     prefixMap.put("p", new ArrayList<Cell>());
143     prefixMap.put("q", new ArrayList<Cell>());
144     prefixMap.put("s", new ArrayList<Cell>());
145 
146     String valueString = "ValueString";
147 
148     for (String row: rows) {
149       Put p = new Put(Bytes.toBytes(row));
150       p.setDurability(Durability.SKIP_WAL);
151       for (String column: columns) {
152         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
153           double rand = Math.random();
154           Cell kv;
155           if (rand < 0.5)
156             kv = KeyValueTestUtil.create(row, family1, column, timestamp,
157                 valueString);
158           else
159             kv = KeyValueTestUtil.create(row, family2, column, timestamp,
160                 valueString);
161           p.add(kv);
162           kvList.add(kv);
163           for (String s: prefixMap.keySet()) {
164             if (column.startsWith(s)) {
165               prefixMap.get(s).add(kv);
166             }
167           }
168         }
169       }
170       region.put(p);
171     }
172 
173     MultipleColumnPrefixFilter filter;
174     Scan scan = new Scan();
175     scan.setMaxVersions();
176     byte [][] filter_prefix = new byte [2][];
177     filter_prefix[0] = new byte [] {'p'};
178     filter_prefix[1] = new byte [] {'q'};
179 
180     filter = new MultipleColumnPrefixFilter(filter_prefix);
181     scan.setFilter(filter);
182     List<Cell> results = new ArrayList<Cell>();
183     InternalScanner scanner = region.getScanner(scan);
184     while (scanner.next(results))
185       ;
186     assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
187 
188     HRegion.closeHRegion(region);
189   }
190 
191   @Test
testMultipleColumnPrefixFilterWithColumnPrefixFilter()192   public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
193     String family = "Family";
194     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
195     htd.addFamily(new HColumnDescriptor(family));
196     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
197     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
198       getDataTestDir(), TEST_UTIL.getConfiguration(),htd);
199 
200     List<String> rows = generateRandomWords(100, "row");
201     List<String> columns = generateRandomWords(10000, "column");
202     long maxTimestamp = 2;
203 
204     String valueString = "ValueString";
205 
206     for (String row: rows) {
207       Put p = new Put(Bytes.toBytes(row));
208       p.setDurability(Durability.SKIP_WAL);
209       for (String column: columns) {
210         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
211           KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
212               valueString);
213           p.add(kv);
214         }
215       }
216       region.put(p);
217     }
218 
219     MultipleColumnPrefixFilter multiplePrefixFilter;
220     Scan scan1 = new Scan();
221     scan1.setMaxVersions();
222     byte [][] filter_prefix = new byte [1][];
223     filter_prefix[0] = new byte [] {'p'};
224 
225     multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
226     scan1.setFilter(multiplePrefixFilter);
227     List<Cell> results1 = new ArrayList<Cell>();
228     InternalScanner scanner1 = region.getScanner(scan1);
229     while (scanner1.next(results1))
230       ;
231 
232     ColumnPrefixFilter singlePrefixFilter;
233     Scan scan2 = new Scan();
234     scan2.setMaxVersions();
235     singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
236 
237     scan2.setFilter(singlePrefixFilter);
238     List<Cell> results2 = new ArrayList<Cell>();
239     InternalScanner scanner2 = region.getScanner(scan1);
240     while (scanner2.next(results2))
241       ;
242 
243     assertEquals(results1.size(), results2.size());
244 
245     HRegion.closeHRegion(region);
246   }
247 
generateRandomWords(int numberOfWords, String suffix)248   List<String> generateRandomWords(int numberOfWords, String suffix) {
249     Set<String> wordSet = new HashSet<String>();
250     for (int i = 0; i < numberOfWords; i++) {
251       int lengthOfWords = (int) (Math.random()*2) + 1;
252       char[] wordChar = new char[lengthOfWords];
253       for (int j = 0; j < wordChar.length; j++) {
254         wordChar[j] = (char) (Math.random() * 26 + 97);
255       }
256       String word;
257       if (suffix == null) {
258         word = new String(wordChar);
259       } else {
260         word = new String(wordChar) + suffix;
261       }
262       wordSet.add(word);
263     }
264     List<String> wordList = new ArrayList<String>(wordSet);
265     return wordList;
266   }
267 
268 }
269 
270 
271