1// Copyright 2017-2019 Aerospike, Inc. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package aerospike_test 16 17import ( 18 "fmt" 19 "time" 20 21 as "github.com/aerospike/aerospike-client-go" 22 atomic "github.com/aerospike/aerospike-client-go/internal/atomic" 23 24 . "github.com/onsi/ginkgo" 25 . "github.com/onsi/gomega" 26) 27 28// ALL tests are isolated by SetName and Key, which are 50 random characters 29var _ = Describe("predexp operations", func() { 30 31 const keyCount = 1000 32 33 var ns = *namespace 34 var set = "predexp_tests" // The name of the set should be consistent because of predexp_modulo tests, since set name is a part of the digest 35 var wpolicy = as.NewWritePolicy(0, 0) 36 37 starbucks := [][2]float64{ 38 {-122.1708441, 37.4241193}, 39 {-122.1492040, 37.4273569}, 40 {-122.1441078, 37.4268202}, 41 {-122.1251714, 37.4130590}, 42 {-122.0964289, 37.4218102}, 43 {-122.0776641, 37.4158199}, 44 {-122.0943475, 37.4114654}, 45 {-122.1122861, 37.4028493}, 46 {-122.0947230, 37.3909250}, 47 {-122.0831037, 37.3876090}, 48 {-122.0707119, 37.3787855}, 49 {-122.0303178, 37.3882739}, 50 {-122.0464861, 37.3786236}, 51 {-122.0582128, 37.3726980}, 52 {-122.0365083, 37.3676930}, 53 } 54 55 var gaptime int64 56 57 insertRecs := atomic.NewAtomicBool(true) 58 59 BeforeEach(func() { 60 if !insertRecs.Get() { 61 return 62 } 63 64 client.DropIndex(nil, ns, set, "intval") 65 client.DropIndex(nil, ns, set, "strval") 66 67 wpolicy = as.NewWritePolicy(0, 24*60*60) 68 69 for ii := 0; ii < keyCount; ii++ { 70 71 // On iteration 333 we pause for a few mSec and note the 72 // time. Later we can check last_update time for either 73 // side of this gap ... 74 // 75 // Also, we update the WritePolicy to never expire so 76 // records w/ 0 TTL can be counted later. 77 // 78 if ii == 333 { 79 time.Sleep(500 * time.Millisecond) 80 gaptime = time.Now().UnixNano() 81 time.Sleep(500 * time.Millisecond) 82 83 wpolicy = as.NewWritePolicy(0, as.TTLDontExpire) 84 } 85 86 key, err := as.NewKey(ns, set, ii) 87 Expect(err).ToNot(HaveOccurred()) 88 89 lng := -122.0 + (0.01 * float64(ii)) 90 lat := 37.5 + (0.01 * float64(ii)) 91 pointstr := fmt.Sprintf( 92 "{ \"type\": \"Point\", \"coordinates\": [%f, %f] }", 93 lng, lat) 94 95 var regionstr string 96 if ii < len(starbucks) { 97 regionstr = fmt.Sprintf( 98 "{ \"type\": \"AeroCircle\", "+ 99 " \"coordinates\": [[%f, %f], 3000.0 ] }", 100 starbucks[ii][0], starbucks[ii][1]) 101 } else { 102 // Somewhere off Africa ... 103 regionstr = 104 "{ \"type\": \"AeroCircle\", " + 105 " \"coordinates\": [[0.0, 0.0], 3000.0 ] }" 106 } 107 108 // Accumulate prime factors of the index into a list and map. 109 listval := []int{} 110 mapval := map[int]string{} 111 for _, ff := range []int{2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31} { 112 if ii >= ff && ii%ff == 0 { 113 listval = append(listval, ff) 114 mapval[ff] = fmt.Sprintf("0x%04x", ff) 115 } 116 } 117 118 ballast := make([]byte, ii*16) 119 120 bins := as.BinMap{ 121 "intval": ii, 122 "strval": fmt.Sprintf("0x%04x", ii), 123 "modval": ii % 10, 124 "locval": as.NewGeoJSONValue(pointstr), 125 "rgnval": as.NewGeoJSONValue(regionstr), 126 "lstval": listval, 127 "mapval": mapval, 128 "ballast": ballast, 129 } 130 err = client.Put(wpolicy, key, bins) 131 Expect(err).ToNot(HaveOccurred()) 132 } 133 134 idxTask, err := client.CreateIndex(wpolicy, ns, set, "intval", "intval", as.NUMERIC) 135 Expect(err).ToNot(HaveOccurred()) 136 Expect(<-idxTask.OnComplete()).ToNot(HaveOccurred()) 137 138 idxTask, err = client.CreateIndex(wpolicy, ns, set, "strval", "strval", as.STRING) 139 Expect(err).ToNot(HaveOccurred()) 140 Expect(<-idxTask.OnComplete()).ToNot(HaveOccurred()) 141 142 insertRecs.Set(false) 143 }) 144 145 // AfterEach(func() { 146 // Expect(client.DropIndex(nil, ns, set, "intval")).ToNot(HaveOccurred()) 147 // Expect(client.DropIndex(nil, ns, set, "strval")).ToNot(HaveOccurred()) 148 // }) 149 150 It("server error with top level predexp value node", func() { 151 152 // This statement doesn't form a predicate expression. 153 stm := as.NewStatement(ns, set) 154 stm.SetFilter(as.NewRangeFilter("intval", 0, 400)) 155 stm.SetPredExp(as.NewPredExpIntegerValue(8)) 156 recordset, err := client.Query(nil, stm) 157 Expect(err).ToNot(HaveOccurred()) 158 for res := range recordset.Results() { 159 Expect(res.Err).To(HaveOccurred()) 160 } 161 }) 162 163 It("server error with multiple top-level predexp", func() { 164 165 stm := as.NewStatement(ns, set) 166 stm.SetFilter(as.NewRangeFilter("intval", 0, 400)) 167 stm.SetPredExp( 168 as.NewPredExpIntegerBin("modval"), 169 as.NewPredExpIntegerValue(8), 170 as.NewPredExpIntegerGreaterEq(), 171 as.NewPredExpIntegerBin("modval"), 172 as.NewPredExpIntegerValue(8), 173 as.NewPredExpIntegerGreaterEq(), 174 ) 175 recordset, err := client.Query(nil, stm) 176 Expect(err).ToNot(HaveOccurred()) 177 for res := range recordset.Results() { 178 Expect(res.Err).To(HaveOccurred()) 179 } 180 }) 181 182 It("server error with missing child predexp", func() { 183 184 stm := as.NewStatement(ns, set) 185 stm.SetFilter(as.NewRangeFilter("intval", 0, 400)) 186 stm.SetPredExp( 187 as.NewPredExpIntegerValue(8), 188 as.NewPredExpIntegerGreaterEq(), 189 ) // needs two children! 190 recordset, err := client.Query(nil, stm) 191 Expect(err).ToNot(HaveOccurred()) 192 for res := range recordset.Results() { 193 Expect(res.Err).To(HaveOccurred()) 194 } 195 }) 196 197 It("predexp must additionally filter indexed query results", func() { 198 199 stm := as.NewStatement(ns, set) 200 stm.SetFilter(as.NewRangeFilter("intval", 0, 400)) 201 stm.SetPredExp( 202 as.NewPredExpIntegerBin("modval"), 203 as.NewPredExpIntegerValue(8), 204 as.NewPredExpIntegerGreaterEq(), 205 ) 206 recordset, err := client.Query(nil, stm) 207 Expect(err).ToNot(HaveOccurred()) 208 209 // The query clause selects [0, 1, ... 400, 401] The predexp 210 // only takes mod 8 and 9, should be 2 pre decade or 80 total. 211 212 cnt := 0 213 for res := range recordset.Results() { 214 Expect(res.Err).ToNot(HaveOccurred()) 215 cnt++ 216 } 217 218 Expect(cnt).To(BeNumerically("==", 80)) 219 }) 220 221 It("predexp must work with implied scan", func() { 222 223 stm := as.NewStatement(ns, set) 224 stm.SetPredExp( 225 as.NewPredExpStringValue("0x0001"), 226 as.NewPredExpStringBin("strval"), 227 as.NewPredExpStringEqual(), 228 ) 229 recordset, err := client.Query(nil, stm) 230 Expect(err).ToNot(HaveOccurred()) 231 232 cnt := 0 233 for res := range recordset.Results() { 234 Expect(res.Err).ToNot(HaveOccurred()) 235 cnt++ 236 } 237 238 Expect(cnt).To(BeNumerically("==", 1)) 239 }) 240 241 It("predexp and or and not must all work", func() { 242 243 stm := as.NewStatement(ns, set) 244 245 // This returns 999 246 stm.SetPredExp( 247 as.NewPredExpStringValue("0x0001"), 248 as.NewPredExpStringBin("strval"), 249 as.NewPredExpStringEqual(), 250 as.NewPredExpNot(), 251 252 // This is two per decade 253 as.NewPredExpIntegerBin("modval"), 254 as.NewPredExpIntegerValue(8), 255 as.NewPredExpIntegerGreaterEq(), 256 257 // Should be 200 258 as.NewPredExpAnd(2), 259 260 // Should exactly match 3 values not in prior set 261 as.NewPredExpStringValue("0x0104"), 262 as.NewPredExpStringBin("strval"), 263 as.NewPredExpStringEqual(), 264 as.NewPredExpStringValue("0x0105"), 265 as.NewPredExpStringBin("strval"), 266 as.NewPredExpStringEqual(), 267 as.NewPredExpStringValue("0x0106"), 268 as.NewPredExpStringBin("strval"), 269 as.NewPredExpStringEqual(), 270 271 // 200 + 3 272 as.NewPredExpOr(4), 273 ) 274 275 recordset, err := client.Query(nil, stm) 276 Expect(err).ToNot(HaveOccurred()) 277 278 cnt := 0 279 for res := range recordset.Results() { 280 Expect(res.Err).ToNot(HaveOccurred()) 281 cnt++ 282 } 283 284 Expect(cnt).To(BeNumerically("==", 203)) 285 }) 286 287 It("predexp regex match must work", func() { 288 289 stm := as.NewStatement(ns, set) 290 stm.SetPredExp( 291 as.NewPredExpStringBin("strval"), 292 as.NewPredExpStringValue("0x00.[12]"), 293 as.NewPredExpStringRegex(0), 294 ) 295 recordset, err := client.Query(nil, stm) 296 Expect(err).ToNot(HaveOccurred()) 297 298 // Should be 32 results: 299 // 0x0001, 0x0002, 300 // 0x0011, 0x0012, 301 // ... 302 // 0x00f1, 0x00f2, 303 304 cnt := 0 305 for res := range recordset.Results() { 306 Expect(res.Err).ToNot(HaveOccurred()) 307 cnt++ 308 } 309 310 Expect(cnt).To(BeNumerically("==", 32)) 311 }) 312 313 It("predexp geo PIR query must work", func() { 314 315 region := 316 "{ " + 317 " \"type\": \"Polygon\", " + 318 " \"coordinates\": [ " + 319 " [[-122.500000, 37.000000],[-121.000000, 37.000000], " + 320 " [-121.000000, 38.080000],[-122.500000, 38.080000], " + 321 " [-122.500000, 37.000000]] " + 322 " ] " + 323 "}" 324 325 stm := as.NewStatement(ns, set) 326 stm.SetPredExp( 327 as.NewPredExpGeoJSONBin("locval"), 328 as.NewPredExpGeoJSONValue(region), 329 as.NewPredExpGeoJSONWithin(), 330 ) 331 recordset, err := client.Query(nil, stm) 332 Expect(err).ToNot(HaveOccurred()) 333 334 cnt := 0 335 for res := range recordset.Results() { 336 Expect(res.Err).ToNot(HaveOccurred()) 337 cnt++ 338 } 339 340 // Correct answer is 59. 341 Expect(cnt).To(BeNumerically("==", 59)) 342 }) 343 344 It("predexp geo RCP query must work", func() { 345 346 point := 347 "{ " + 348 " \"type\": \"Point\", " + 349 " \"coordinates\": [ -122.0986857, 37.4214209 ] " + 350 "}" 351 352 stm := as.NewStatement(ns, set) 353 stm.SetPredExp( 354 as.NewPredExpGeoJSONBin("rgnval"), 355 as.NewPredExpGeoJSONValue(point), 356 as.NewPredExpGeoJSONContains(), 357 ) 358 recordset, err := client.Query(nil, stm) 359 Expect(err).ToNot(HaveOccurred()) 360 361 // Correct answer is 6. See: 362 // aerospike-client-c/src/test/aerospike_geo/query_geospatial.c: 363 // predexp_points_within_region 364 365 cnt := 0 366 for res := range recordset.Results() { 367 Expect(res.Err).ToNot(HaveOccurred()) 368 cnt++ 369 } 370 371 // Correct answer is 5. See: 372 // aerospike-client-c/src/test/aerospike_geo/query_geospatial.c: 373 // predexp_regions_containing_point 374 375 Expect(cnt).To(BeNumerically("==", 5)) 376 }) 377 378 It("predexp last_update must work", func() { 379 380 stm := as.NewStatement(ns, set) 381 stm.SetPredExp(as.NewPredExpRecLastUpdate(), 382 as.NewPredExpIntegerValue(gaptime), 383 as.NewPredExpIntegerGreater(), 384 ) 385 recordset, err := client.Query(nil, stm) 386 Expect(err).ToNot(HaveOccurred()) 387 388 cnt := 0 389 for res := range recordset.Results() { 390 Expect(res.Err).ToNot(HaveOccurred()) 391 cnt++ 392 } 393 394 // The answer should be 1000 - 333 = 667 395 396 Expect(cnt).To(BeNumerically("==", 667)) 397 }) 398 399 It("predexp void_time must work", func() { 400 401 stm := as.NewStatement(ns, set) 402 stm.SetPredExp( 403 as.NewPredExpIntegerValue(0), 404 as.NewPredExpRecVoidTime(), 405 as.NewPredExpIntegerEqual(), 406 ) 407 recordset, err := client.Query(nil, stm) 408 Expect(err).ToNot(HaveOccurred()) 409 410 cnt := 0 411 for res := range recordset.Results() { 412 Expect(res.Err).ToNot(HaveOccurred()) 413 cnt++ 414 } 415 416 // The answer should be 1000 - 333 = 667 417 418 Expect(cnt).To(BeNumerically("==", 667)) 419 }) 420 421 It("predexp rec_size work", func() { 422 423 if len(nsInfo(ns, "device_total_bytes")) == 0 { 424 Skip("Skipping Predexp rec_size test since the namespace is not persisted.") 425 } 426 427 stm := as.NewStatement(ns, set) 428 stm.SetPredExp( 429 as.NewPredExpRecDeviceSize(), 430 as.NewPredExpIntegerValue(12*1024), 431 as.NewPredExpIntegerGreaterEq(), 432 ) 433 recordset, err := client.Query(nil, stm) 434 Expect(err).ToNot(HaveOccurred()) 435 436 cnt := 0 437 for res := range recordset.Results() { 438 Expect(res.Err).ToNot(HaveOccurred()) 439 cnt++ 440 } 441 442 // Answer should roughly be 1000 - (12/16 * 1000) ~= 250 + ovhd 443 444 Expect(cnt).To(BeNumerically(">", 250)) 445 Expect(cnt).To(BeNumerically("<", 300)) 446 }) 447 448 It("predexp digest_modulo must work", func() { 449 450 cnt := []int{0, 0, 0} 451 for _, ndx := range []int64{0, 1, 2} { 452 stm := as.NewStatement(ns, set) 453 stm.SetPredExp( 454 as.NewPredExpRecDigestModulo(3), 455 as.NewPredExpIntegerValue(ndx), 456 as.NewPredExpIntegerEqual(), 457 ) 458 recordset, err := client.Query(nil, stm) 459 Expect(err).ToNot(HaveOccurred()) 460 461 for res := range recordset.Results() { 462 Expect(res.Err).ToNot(HaveOccurred()) 463 cnt[ndx]++ 464 } 465 } 466 467 // The count should be split 3 ways, roughly equally. 468 sum := 0 469 Expect(cnt).To(Equal([]int{308, 374, 318})) 470 for _, cc := range cnt { 471 sum += cc 472 } 473 Expect(sum).To(BeNumerically("==", 1000)) 474 }) 475 476 It("predexp list_iter_or work", func() { 477 478 // Select all records w/ list contains a 17. 479 480 stm := as.NewStatement(ns, set) 481 stm.SetPredExp( 482 as.NewPredExpIntegerValue(17), 483 as.NewPredExpIntegerVar("ff"), 484 as.NewPredExpIntegerEqual(), 485 as.NewPredExpListBin("lstval"), 486 as.NewPredExpListIterateOr("ff"), 487 ) 488 recordset, err := client.Query(nil, stm) 489 Expect(err).ToNot(HaveOccurred()) 490 491 cnt := 0 492 for res := range recordset.Results() { 493 Expect(res.Err).ToNot(HaveOccurred()) 494 cnt++ 495 } 496 497 // Answer should be floor(1000 / 17) = 58 498 499 Expect(cnt).To(BeNumerically("==", 58)) 500 }) 501 502 It("predexp list_iter_and work", func() { 503 504 // Select all records w/ list doesn't have a 3. 505 506 stm := as.NewStatement(ns, set) 507 stm.SetPredExp( 508 as.NewPredExpIntegerValue(3), 509 as.NewPredExpIntegerVar("ff"), 510 as.NewPredExpIntegerEqual(), 511 as.NewPredExpNot(), 512 as.NewPredExpListBin("lstval"), 513 as.NewPredExpListIterateAnd("ff"), 514 ) 515 recordset, err := client.Query(nil, stm) 516 Expect(err).ToNot(HaveOccurred()) 517 518 cnt := 0 519 for res := range recordset.Results() { 520 Expect(res.Err).ToNot(HaveOccurred()) 521 cnt++ 522 } 523 524 // Answer should be 1000 - (ceil(1000 / 3) - 1) = 667 525 526 Expect(cnt).To(BeNumerically("==", 667)) 527 }) 528 529 It("predexp mapkey_iter_or work", func() { 530 531 // Select all records w/ mapkey containing 19. 532 533 stm := as.NewStatement(ns, set) 534 stm.SetPredExp( 535 as.NewPredExpIntegerValue(19), 536 as.NewPredExpIntegerVar("kk"), 537 as.NewPredExpIntegerEqual(), 538 as.NewPredExpMapBin("mapval"), 539 as.NewPredExpMapKeyIterateOr("kk"), 540 ) 541 recordset, err := client.Query(nil, stm) 542 Expect(err).ToNot(HaveOccurred()) 543 544 cnt := 0 545 for res := range recordset.Results() { 546 Expect(res.Err).ToNot(HaveOccurred()) 547 cnt++ 548 } 549 550 // Answer should be floor(1000 / 19) = 52 551 552 Expect(cnt).To(BeNumerically("==", 52)) 553 }) 554 555 It("predexp mapkey_iter_and work", func() { 556 557 // Select all records w/ no mapkey containing 5. 558 559 stm := as.NewStatement(ns, set) 560 stm.SetPredExp( 561 as.NewPredExpIntegerValue(5), 562 as.NewPredExpIntegerVar("kk"), 563 as.NewPredExpIntegerEqual(), 564 as.NewPredExpNot(), 565 as.NewPredExpMapBin("mapval"), 566 as.NewPredExpMapKeyIterateAnd("kk"), 567 ) 568 recordset, err := client.Query(nil, stm) 569 Expect(err).ToNot(HaveOccurred()) 570 571 cnt := 0 572 for res := range recordset.Results() { 573 Expect(res.Err).ToNot(HaveOccurred()) 574 cnt++ 575 } 576 577 // Answer should be 1000 - (ceil(1000 / 5) - 1) = 801 578 579 Expect(cnt).To(BeNumerically("==", 801)) 580 }) 581 582 It("predexp mapval_iter_or work", func() { 583 584 // Select all records w/ mapval of 19 ("0x0013") 585 586 stm := as.NewStatement(ns, set) 587 stm.SetPredExp( 588 as.NewPredExpStringValue("0x0013"), 589 as.NewPredExpStringVar("vv"), 590 as.NewPredExpStringEqual(), 591 as.NewPredExpMapBin("mapval"), 592 as.NewPredExpMapValIterateOr("vv"), 593 ) 594 recordset, err := client.Query(nil, stm) 595 Expect(err).ToNot(HaveOccurred()) 596 597 cnt := 0 598 for res := range recordset.Results() { 599 Expect(res.Err).ToNot(HaveOccurred()) 600 cnt++ 601 } 602 603 // Answer should be floor(1000 / 19) = 52 604 605 Expect(cnt).To(BeNumerically("==", 52)) 606 }) 607 608 It("predexp mapval_iter_and work", func() { 609 610 // Select all records w/ no mapval of 5 ("0x0005"). 611 612 stm := as.NewStatement(ns, set) 613 stm.SetPredExp( 614 as.NewPredExpStringValue("0x0005"), 615 as.NewPredExpStringVar("vv"), 616 as.NewPredExpStringEqual(), 617 as.NewPredExpNot(), 618 as.NewPredExpMapBin("mapval"), 619 as.NewPredExpMapValIterateAnd("vv"), 620 ) 621 recordset, err := client.Query(nil, stm) 622 Expect(err).ToNot(HaveOccurred()) 623 624 cnt := 0 625 for res := range recordset.Results() { 626 Expect(res.Err).ToNot(HaveOccurred()) 627 cnt++ 628 } 629 630 // Answer should be 1000 - (ceil(1000 / 5) - 1) = 801 631 632 Expect(cnt).To(BeNumerically("==", 801)) 633 }) 634 635}) 636