1 // This file is part of OpenTSDB. 2 // Copyright (C) 2015 The OpenTSDB Authors. 3 // 4 // This program is free software: you can redistribute it and/or modify it 5 // under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 2.1 of the License, or (at your 7 // option) any later version. This program is distributed in the hope that it 8 // will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 9 // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License for more details. You should have received a copy 11 // of the GNU Lesser General Public License along with this program. If not, 12 // see <http://www.gnu.org/licenses/>. 13 package net.opentsdb.query.filter; 14 15 import static org.junit.Assert.assertArrayEquals; 16 import static org.junit.Assert.assertEquals; 17 import static org.junit.Assert.assertFalse; 18 import static org.junit.Assert.assertNotSame; 19 import static org.junit.Assert.assertNull; 20 import static org.junit.Assert.assertTrue; 21 import static org.junit.Assert.fail; 22 23 import java.util.ArrayList; 24 import java.util.HashMap; 25 import java.util.List; 26 import java.util.Map; 27 import java.util.Set; 28 29 import net.opentsdb.core.BaseTsdbTest; 30 import net.opentsdb.core.TSDB; 31 import net.opentsdb.uid.NoSuchUniqueName; 32 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.powermock.core.classloader.annotations.PowerMockIgnore; 36 import org.powermock.core.classloader.annotations.PrepareForTest; 37 import org.powermock.modules.junit4.PowerMockRunner; 38 import org.powermock.reflect.Whitebox; 39 40 import com.stumbleupon.async.DeferredGroupException; 41 42 @RunWith(PowerMockRunner.class) 43 @PowerMockIgnore({"javax.management.*", "javax.xml.*", 44 "ch.qos.*", "org.slf4j.*", 45 "com.sum.*", "org.xml.*"}) 46 @PrepareForTest({ TSDB.class }) 47 public class TestTagVFilter extends BaseTsdbTest { 48 49 @Test (expected = IllegalArgumentException.class) getFilterNullTagk()50 public void getFilterNullTagk() throws Exception { 51 TagVFilter.getFilter(null, "myflter"); 52 } 53 54 @Test (expected = IllegalArgumentException.class) getFilterEmptyTagk()55 public void getFilterEmptyTagk() throws Exception { 56 TagVFilter.getFilter(null, "myflter"); 57 } 58 59 @Test (expected = IllegalArgumentException.class) getFilterEmptyFilter()60 public void getFilterEmptyFilter() throws Exception { 61 TagVFilter.getFilter(TAGK_STRING, ""); 62 } 63 64 @Test (expected = IllegalArgumentException.class) getFilterNullFilter()65 public void getFilterNullFilter() throws Exception { 66 TagVFilter.getFilter(TAGK_STRING, null); 67 } 68 69 @Test getFilterGroupBy()70 public void getFilterGroupBy() throws Exception { 71 assertNull(TagVFilter.getFilter(TAGK_STRING, "*")); 72 } 73 74 @Test getFilterLiteral()75 public void getFilterLiteral() throws Exception { 76 assertNull(TagVFilter.getFilter(TAGK_STRING, TAGV_STRING)); 77 } 78 79 @Test getFilterGroupByPiped()80 public void getFilterGroupByPiped() throws Exception { 81 assertNull(TagVFilter.getFilter(TAGK_STRING, "web01|web02")); 82 } 83 84 @Test getFilterWildcard()85 public void getFilterWildcard() throws Exception { 86 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 87 TagVWildcardFilter.FILTER_NAME + "(*bonk.com)"); 88 assertEquals(TAGK_STRING, filter.getTagk()); 89 assertTrue(filter instanceof TagVWildcardFilter); 90 assertFalse(((TagVWildcardFilter)filter).isCaseInsensitive()); 91 } 92 93 @Test getFilterWildcardInsensitive()94 public void getFilterWildcardInsensitive() throws Exception { 95 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 96 TagVWildcardFilter.TagVIWildcardFilter.FILTER_NAME + "(*bonk.com)"); 97 assertEquals(TAGK_STRING, filter.getTagk()); 98 assertTrue(filter instanceof TagVWildcardFilter); 99 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 100 } 101 102 @Test getFilterWildcardFatfinger()103 public void getFilterWildcardFatfinger() throws Exception { 104 // falls through to the shortcut 105 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 106 "wil@*sugarbean"); 107 assertEquals(TAGK_STRING, filter.getTagk()); 108 assertTrue(filter instanceof TagVWildcardFilter); 109 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 110 } 111 112 @Test getFilterWildcardImplicit()113 public void getFilterWildcardImplicit() throws Exception { 114 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, "*bonk.com"); 115 assertEquals(TAGK_STRING, filter.getTagk()); 116 assertTrue(filter instanceof TagVWildcardFilter); 117 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 118 } 119 120 @Test getFilterPipe()121 public void getFilterPipe() throws Exception { 122 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 123 TagVLiteralOrFilter.FILTER_NAME + "(quirm|bonk)"); 124 assertEquals(TAGK_STRING, filter.getTagk()); 125 assertTrue(filter instanceof TagVLiteralOrFilter); 126 assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); 127 } 128 129 @Test getFilterPipeInsensitive()130 public void getFilterPipeInsensitive() throws Exception { 131 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 132 TagVLiteralOrFilter.TagVILiteralOrFilter.FILTER_NAME + "(quirm|bonk)"); 133 assertEquals(TAGK_STRING, filter.getTagk()); 134 assertTrue(filter instanceof TagVLiteralOrFilter); 135 assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); 136 } 137 138 @Test getFilterPipeFatfinger()139 public void getFilterPipeFatfinger() throws Exception { 140 assertNull(TagVFilter.getFilter(TAGK_STRING, "lite@sugarbean|granny")); 141 } 142 143 @Test getFilterRegex()144 public void getFilterRegex() throws Exception { 145 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 146 TagVRegexFilter.FILTER_NAME + "(.*sugarbean)"); 147 assertEquals(TAGK_STRING, filter.getTagk()); 148 assertTrue(filter instanceof TagVRegexFilter); 149 } 150 151 @Test getFilterRegexFatFinger()152 public void getFilterRegexFatFinger() throws Exception { 153 // falls through to the implicity 154 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, "rexp@.*sugarbean"); 155 assertEquals(TAGK_STRING, filter.getTagk()); 156 assertTrue(filter instanceof TagVWildcardFilter); 157 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 158 } 159 160 @Test getFilterRegexCase()161 public void getFilterRegexCase() throws Exception { 162 final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, 163 TagVRegexFilter.FILTER_NAME.toUpperCase() + "(.*sugarbean)"); 164 assertEquals(TAGK_STRING, filter.getTagk()); 165 assertTrue(filter instanceof TagVRegexFilter); 166 } 167 168 @Test (expected = IllegalArgumentException.class) getFilterMissingClosingParens()169 public void getFilterMissingClosingParens() throws Exception { 170 TagVFilter.getFilter(TAGK_STRING, TagVRegexFilter.FILTER_NAME + "(.*sugarbean"); 171 } 172 173 @Test (expected = IllegalArgumentException.class) getFilterEmptyParens()174 public void getFilterEmptyParens() throws Exception { 175 TagVFilter.getFilter(TAGK_STRING, TagVRegexFilter.FILTER_NAME + "()"); 176 } 177 178 @Test (expected = IllegalArgumentException.class) getFilterUnknownType()179 public void getFilterUnknownType() throws Exception { 180 TagVFilter.getFilter(TAGK_STRING, "dummyfilter(nothere)"); 181 } 182 183 @Test resolveName()184 public void resolveName() throws Exception { 185 final TagVFilter filter = new TagVWildcardFilter(TAGK_STRING, "*omnia"); 186 filter.resolveTagkName(tsdb).join(); 187 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 188 assertTrue(filter.getTagVUids().isEmpty()); 189 } 190 191 @Test resolveNameLiteral()192 public void resolveNameLiteral() throws Exception { 193 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, TAGV_STRING); 194 filter.resolveTagkName(tsdb).join(); 195 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 196 assertEquals(1, filter.getTagVUids().size()); 197 assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); 198 } 199 200 @Test resolveNameLiterals()201 public void resolveNameLiterals() throws Exception { 202 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02"); 203 filter.resolveTagkName(tsdb).join(); 204 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 205 assertEquals(2, filter.getTagVUids().size()); 206 assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); 207 assertArrayEquals(TAGV_B_BYTES, filter.getTagVUids().get(1)); 208 } 209 210 @Test (expected = DeferredGroupException.class) resolveNameLiteralsNSUNTagV()211 public void resolveNameLiteralsNSUNTagV() throws Exception { 212 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web03"); 213 filter.resolveTagkName(tsdb).join(); 214 } 215 216 @Test resolveNameLiteralsNSUNTagvSkipped()217 public void resolveNameLiteralsNSUNTagvSkipped() throws Exception { 218 config.overrideConfig("tsd.query.skip_unresolved_tagvs", "true"); 219 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web03"); 220 filter.resolveTagkName(tsdb).join(); 221 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 222 assertEquals(1, filter.getTagVUids().size()); 223 assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); 224 } 225 226 @Test resolveNameLiteralsTooMany()227 public void resolveNameLiteralsTooMany() throws Exception { 228 config.overrideConfig("tsd.query.filter.expansion_limit", "1"); 229 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02"); 230 filter.resolveTagkName(tsdb).join(); 231 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 232 assertTrue(filter.getTagVUids().isEmpty()); 233 } 234 235 @Test resolveNameLiteralsCaseInsensitive()236 public void resolveNameLiteralsCaseInsensitive() throws Exception { 237 final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02", 238 true); 239 filter.resolveTagkName(tsdb).join(); 240 assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); 241 assertTrue(filter.getTagVUids().isEmpty()); 242 } 243 244 @Test (expected = NoSuchUniqueName.class) resolveNameNSUN()245 public void resolveNameNSUN() throws Exception { 246 final TagVFilter filter = new TagVWildcardFilter(NSUN_TAGK, "*omnia"); 247 filter.resolveTagkName(tsdb).join(); 248 } 249 250 @Test (expected = NullPointerException.class) resolveNameNullTSDB()251 public void resolveNameNullTSDB() throws Exception { 252 new TagVWildcardFilter("host", "*omnia").resolveTagkName(null); 253 } 254 255 @Test comparableTest()256 public void comparableTest() throws Exception { 257 final TagVFilter filter_a = new TagVWildcardFilter("host", "*omnia"); 258 Whitebox.setInternalState(filter_a, "tagk_bytes", new byte[] { 0, 0, 0, 1 }); 259 final TagVFilter filter_b = new TagVRegexFilter("dc", ".*katch"); 260 Whitebox.setInternalState(filter_b, "tagk_bytes", new byte[] { 0, 0, 0, 2 }); 261 262 assertEquals(0, filter_a.compareTo(filter_a)); 263 assertEquals(-1, filter_a.compareTo(filter_b)); 264 assertEquals(1, filter_b.compareTo(filter_a)); 265 266 Whitebox.setInternalState(filter_a, "tagk_bytes", (byte[])null); 267 assertEquals(0, filter_a.compareTo(filter_a)); 268 assertEquals(-1, filter_a.compareTo(filter_b)); 269 assertEquals(1, filter_b.compareTo(filter_a)); 270 271 Whitebox.setInternalState(filter_b, "tagk_bytes", (byte[])null); 272 assertEquals(0, filter_a.compareTo(filter_a)); 273 assertEquals(0, filter_a.compareTo(filter_b)); 274 assertEquals(0, filter_b.compareTo(filter_a)); 275 276 } 277 278 @Test stripParentheses()279 public void stripParentheses() throws Exception { 280 assertEquals(".*sugarbean", TagVFilter.stripParentheses( 281 TagVRegexFilter.FILTER_NAME + "(.*sugarbean)")); 282 } 283 284 @Test stripParenthesesEmptyParentheses()285 public void stripParenthesesEmptyParentheses() throws Exception { 286 // let the filter's ctor handle this case 287 assertEquals("", TagVFilter.stripParentheses( 288 TagVRegexFilter.FILTER_NAME + "()")); 289 } 290 291 @Test (expected = IllegalArgumentException.class) stripParenthesesMissingClosing()292 public void stripParenthesesMissingClosing() throws Exception { 293 TagVFilter.stripParentheses(TagVRegexFilter.FILTER_NAME + "(.*sugarbean"); 294 } 295 296 @Test (expected = IllegalArgumentException.class) stripParenthesesMissingOpening()297 public void stripParenthesesMissingOpening() throws Exception { 298 TagVFilter.stripParentheses("regexp.*sugarbean)"); 299 } 300 301 @Test (expected = IllegalArgumentException.class) stripParenthesesNull()302 public void stripParenthesesNull() throws Exception { 303 TagVFilter.stripParentheses(null); 304 } 305 306 @Test (expected = IllegalArgumentException.class) stripParenthesesEmpty()307 public void stripParenthesesEmpty() throws Exception { 308 TagVFilter.stripParentheses(""); 309 } 310 311 @SuppressWarnings("unchecked") 312 @Test tagsToFiltersOldGroupBy()313 public void tagsToFiltersOldGroupBy() throws Exception { 314 final Map<String, String> tags = new HashMap<String, String>(3); 315 tags.put("host", "quirm"); // literal 316 tags.put("owner", "vimes|vetinary"); // pipe 317 tags.put("colo", "*"); // group by all 318 final List<TagVFilter> filters = new ArrayList<TagVFilter>(3); 319 TagVFilter.tagsToFilters(tags, filters); 320 321 assertEquals(3, filters.size()); 322 for (final TagVFilter filter : filters) { 323 if (filter.getTagk().equals("host")) { 324 assertTrue(filter instanceof TagVLiteralOrFilter); 325 assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); 326 assertEquals(1, ((Set<String>)Whitebox 327 .getInternalState(filter, "literals")).size()); 328 } else if (filter.getTagk().equals("owner")) { 329 assertTrue(filter instanceof TagVLiteralOrFilter); 330 assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); 331 assertEquals(2, ((Set<String>)Whitebox 332 .getInternalState(filter, "literals")).size()); 333 } else if (filter.getTagk().equals("colo")) { 334 assertTrue(filter instanceof TagVWildcardFilter); 335 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 336 } else { 337 fail("Unexpected filter type: " + filter); 338 } 339 assertTrue(filter.isGroupBy()); 340 } 341 } 342 343 @Test tagsToFiltersNewFunctions()344 public void tagsToFiltersNewFunctions() throws Exception { 345 final Map<String, String> tags = new HashMap<String, String>(4); 346 tags.put("host", "*beybi"); 347 tags.put("owner", "wildcard(*snapcase*)"); 348 tags.put("colo", "regexp(.*opolis)"); 349 tags.put("geo", "literal_or(tsort|chalk)"); 350 final List<TagVFilter> filters = new ArrayList<TagVFilter>(3); 351 TagVFilter.tagsToFilters(tags, filters); 352 353 assertEquals(4, filters.size()); 354 for (final TagVFilter filter : filters) { 355 if (filter.getTagk().equals("host")) { 356 assertTrue(filter instanceof TagVWildcardFilter); 357 assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); 358 } else if (filter.getTagk().equals("owner")) { 359 assertTrue(filter instanceof TagVWildcardFilter); 360 assertFalse(((TagVWildcardFilter)filter).isCaseInsensitive()); 361 } else if (filter.getTagk().equals("colo")) { 362 assertTrue(filter instanceof TagVRegexFilter); 363 } else if (filter.getTagk().equals("geo")) { 364 assertTrue(filter instanceof TagVLiteralOrFilter); 365 assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); 366 } else { 367 fail("Unexpected filter type: " + filter); 368 } 369 assertTrue(filter.isGroupBy()); 370 } 371 } 372 373 @Test (expected = IllegalArgumentException.class) tagsToFiltersNoSuchFunction()374 public void tagsToFiltersNoSuchFunction() throws Exception { 375 final Map<String, String> tags = new HashMap<String, String>(1); 376 tags.put("host", "doesnotexist(*beybi)"); 377 final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); 378 TagVFilter.tagsToFilters(tags, filters); 379 } 380 381 @Test tagsToFiltersDuplicate()382 public void tagsToFiltersDuplicate() throws Exception { 383 final Map<String, String> tags = new HashMap<String, String>(1); 384 tags.put("host", "*beybi"); 385 final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); 386 filters.add(new TagVWildcardFilter("host", "*beybi", true)); 387 assertFalse(filters.get(0).isGroupBy()); 388 TagVFilter.tagsToFilters(tags, filters); 389 assertEquals(1, filters.size()); 390 assertTrue(filters.get(0).isGroupBy()); 391 } 392 393 @Test tagsToFiltersSameTagDiffValues()394 public void tagsToFiltersSameTagDiffValues() throws Exception { 395 final Map<String, String> tags = new HashMap<String, String>(1); 396 tags.put("host", "*beybi"); 397 final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); 398 filters.add(new TagVWildcardFilter("host", "*helit", true)); 399 assertFalse(filters.get(0).isGroupBy()); 400 TagVFilter.tagsToFilters(tags, filters); 401 assertEquals(2, filters.size()); 402 } 403 404 @Test getCopy()405 public void getCopy() { 406 final TagVFilter filter = TagVFilter.Builder() 407 .setFilter("*") 408 .setTagk(TAGK_STRING) 409 .setType("wildcard") 410 .setGroupBy(true) 411 .build(); 412 final TagVFilter copy = filter.getCopy(); 413 assertNotSame(filter, copy); 414 assertEquals(filter.filter, copy.filter); 415 assertEquals(filter.tagk, copy.tagk); 416 assertEquals(filter.getType(), copy.getType()); 417 assertEquals(filter.group_by, copy.group_by); 418 } 419 420 // TODO - test the plugin loader similar to the other plugins 421 } 422