1// Copyright (c) 2014 Couchbase, Inc. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bleve 16 17import ( 18 "fmt" 19 "io/ioutil" 20 "log" 21 "math" 22 "os" 23 "reflect" 24 "sort" 25 "strconv" 26 "strings" 27 "sync" 28 "testing" 29 "time" 30 31 "golang.org/x/net/context" 32 33 "github.com/blevesearch/bleve/analysis/analyzer/keyword" 34 "github.com/blevesearch/bleve/document" 35 "github.com/blevesearch/bleve/index" 36 "github.com/blevesearch/bleve/index/store/null" 37 "github.com/blevesearch/bleve/mapping" 38 "github.com/blevesearch/bleve/search" 39 "github.com/blevesearch/bleve/search/query" 40) 41 42func TestCrud(t *testing.T) { 43 defer func() { 44 err := os.RemoveAll("testidx") 45 if err != nil { 46 t.Fatal(err) 47 } 48 }() 49 50 index, err := New("testidx", NewIndexMapping()) 51 if err != nil { 52 t.Fatal(err) 53 } 54 55 doca := map[string]interface{}{ 56 "name": "marty", 57 "desc": "gophercon india", 58 } 59 err = index.Index("a", doca) 60 if err != nil { 61 t.Error(err) 62 } 63 64 docy := map[string]interface{}{ 65 "name": "jasper", 66 "desc": "clojure", 67 } 68 err = index.Index("y", docy) 69 if err != nil { 70 t.Error(err) 71 } 72 73 err = index.Delete("y") 74 if err != nil { 75 t.Error(err) 76 } 77 78 docx := map[string]interface{}{ 79 "name": "rose", 80 "desc": "googler", 81 } 82 err = index.Index("x", docx) 83 if err != nil { 84 t.Error(err) 85 } 86 87 err = index.SetInternal([]byte("status"), []byte("pending")) 88 if err != nil { 89 t.Error(err) 90 } 91 92 docb := map[string]interface{}{ 93 "name": "steve", 94 "desc": "cbft master", 95 } 96 batch := index.NewBatch() 97 err = batch.Index("b", docb) 98 if err != nil { 99 t.Error(err) 100 } 101 batch.Delete("x") 102 batch.SetInternal([]byte("batchi"), []byte("batchv")) 103 batch.DeleteInternal([]byte("status")) 104 err = index.Batch(batch) 105 if err != nil { 106 t.Error(err) 107 } 108 val, err := index.GetInternal([]byte("batchi")) 109 if err != nil { 110 t.Error(err) 111 } 112 if string(val) != "batchv" { 113 t.Errorf("expected 'batchv', got '%s'", val) 114 } 115 val, err = index.GetInternal([]byte("status")) 116 if err != nil { 117 t.Error(err) 118 } 119 if val != nil { 120 t.Errorf("expected nil, got '%s'", val) 121 } 122 123 err = index.SetInternal([]byte("seqno"), []byte("7")) 124 if err != nil { 125 t.Error(err) 126 } 127 err = index.SetInternal([]byte("status"), []byte("ready")) 128 if err != nil { 129 t.Error(err) 130 } 131 err = index.DeleteInternal([]byte("status")) 132 if err != nil { 133 t.Error(err) 134 } 135 val, err = index.GetInternal([]byte("status")) 136 if err != nil { 137 t.Error(err) 138 } 139 if val != nil { 140 t.Errorf("expected nil, got '%s'", val) 141 } 142 143 val, err = index.GetInternal([]byte("seqno")) 144 if err != nil { 145 t.Error(err) 146 } 147 if string(val) != "7" { 148 t.Errorf("expected '7', got '%s'", val) 149 } 150 151 // close the index, open it again, and try some more things 152 err = index.Close() 153 if err != nil { 154 t.Fatal(err) 155 } 156 157 index, err = Open("testidx") 158 if err != nil { 159 t.Fatal(err) 160 } 161 defer func() { 162 err := index.Close() 163 if err != nil { 164 t.Fatal(err) 165 } 166 }() 167 168 count, err := index.DocCount() 169 if err != nil { 170 t.Fatal(err) 171 } 172 if count != 2 { 173 t.Errorf("expected doc count 2, got %d", count) 174 } 175 176 doc, err := index.Document("a") 177 if err != nil { 178 t.Fatal(err) 179 } 180 if doc == nil { 181 t.Errorf("expected doc not nil, got nil") 182 } 183 foundNameField := false 184 for _, field := range doc.Fields { 185 if field.Name() == "name" && string(field.Value()) == "marty" { 186 foundNameField = true 187 } 188 } 189 if !foundNameField { 190 t.Errorf("expected to find field named 'name' with value 'marty'") 191 } 192 193 fields, err := index.Fields() 194 if err != nil { 195 t.Fatal(err) 196 } 197 expectedFields := map[string]bool{ 198 "_all": false, 199 "name": false, 200 "desc": false, 201 } 202 if len(fields) < len(expectedFields) { 203 t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) 204 } 205 for _, f := range fields { 206 expectedFields[f] = true 207 } 208 for ef, efp := range expectedFields { 209 if !efp { 210 t.Errorf("field %s is missing", ef) 211 } 212 } 213} 214 215func TestIndexCreateNewOverExisting(t *testing.T) { 216 defer func() { 217 err := os.RemoveAll("testidx") 218 if err != nil { 219 t.Fatal(err) 220 } 221 }() 222 223 index, err := New("testidx", NewIndexMapping()) 224 if err != nil { 225 t.Fatal(err) 226 } 227 err = index.Close() 228 if err != nil { 229 t.Fatal(err) 230 } 231 index, err = New("testidx", NewIndexMapping()) 232 if err != ErrorIndexPathExists { 233 t.Fatalf("expected error index path exists, got %v", err) 234 } 235} 236 237func TestIndexOpenNonExisting(t *testing.T) { 238 _, err := Open("doesnotexist") 239 if err != ErrorIndexPathDoesNotExist { 240 t.Fatalf("expected error index path does not exist, got %v", err) 241 } 242} 243 244func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) { 245 defer func() { 246 err := os.RemoveAll("testidx") 247 if err != nil { 248 t.Fatal(err) 249 } 250 }() 251 252 index, err := New("testidx", NewIndexMapping()) 253 if err != nil { 254 t.Fatal(err) 255 } 256 err = index.Close() 257 if err != nil { 258 t.Fatal(err) 259 } 260 261 // now intentionally change the storage type 262 err = ioutil.WriteFile("testidx/index_meta.json", []byte(`{"storage":"mystery"}`), 0666) 263 if err != nil { 264 t.Fatal(err) 265 } 266 267 index, err = Open("testidx") 268 if err != ErrorUnknownStorageType { 269 t.Fatalf("expected error unknown storage type, got %v", err) 270 } 271 272 // now intentionally corrupt the metadata 273 err = ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666) 274 if err != nil { 275 t.Fatal(err) 276 } 277 278 index, err = Open("testidx") 279 if err != ErrorIndexMetaCorrupt { 280 t.Fatalf("expected error index metadata corrupted, got %v", err) 281 } 282 283 // now intentionally remove the metadata 284 err = os.Remove("testidx/index_meta.json") 285 if err != nil { 286 t.Fatal(err) 287 } 288 289 index, err = Open("testidx") 290 if err != ErrorIndexMetaMissing { 291 t.Fatalf("expected error index metadata missing, got %v", err) 292 } 293} 294 295func TestInMemIndex(t *testing.T) { 296 297 index, err := NewMemOnly(NewIndexMapping()) 298 if err != nil { 299 t.Fatal(err) 300 } 301 err = index.Close() 302 if err != nil { 303 t.Fatal(err) 304 } 305} 306 307func TestClosedIndex(t *testing.T) { 308 index, err := NewMemOnly(NewIndexMapping()) 309 if err != nil { 310 t.Fatal(err) 311 } 312 err = index.Close() 313 if err != nil { 314 t.Fatal(err) 315 } 316 317 err = index.Index("test", "test") 318 if err != ErrorIndexClosed { 319 t.Errorf("expected error index closed, got %v", err) 320 } 321 322 err = index.Delete("test") 323 if err != ErrorIndexClosed { 324 t.Errorf("expected error index closed, got %v", err) 325 } 326 327 b := index.NewBatch() 328 err = index.Batch(b) 329 if err != ErrorIndexClosed { 330 t.Errorf("expected error index closed, got %v", err) 331 } 332 333 _, err = index.Document("test") 334 if err != ErrorIndexClosed { 335 t.Errorf("expected error index closed, got %v", err) 336 } 337 338 _, err = index.DocCount() 339 if err != ErrorIndexClosed { 340 t.Errorf("expected error index closed, got %v", err) 341 } 342 343 _, err = index.Search(NewSearchRequest(NewTermQuery("test"))) 344 if err != ErrorIndexClosed { 345 t.Errorf("expected error index closed, got %v", err) 346 } 347 348 _, err = index.Fields() 349 if err != ErrorIndexClosed { 350 t.Errorf("expected error index closed, got %v", err) 351 } 352} 353 354type slowQuery struct { 355 actual query.Query 356 delay time.Duration 357} 358 359func (s *slowQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { 360 time.Sleep(s.delay) 361 return s.actual.Searcher(i, m, options) 362} 363 364func TestSlowSearch(t *testing.T) { 365 defer func() { 366 err := os.RemoveAll("testidx") 367 if err != nil { 368 t.Fatal(err) 369 } 370 }() 371 372 defer func() { 373 // reset logger back to normal 374 SetLog(log.New(ioutil.Discard, "bleve", log.LstdFlags)) 375 }() 376 // set custom logger 377 var sdw sawDataWriter 378 SetLog(log.New(&sdw, "bleve", log.LstdFlags)) 379 380 index, err := New("testidx", NewIndexMapping()) 381 if err != nil { 382 t.Fatal(err) 383 } 384 defer func() { 385 err := index.Close() 386 if err != nil { 387 t.Fatal(err) 388 } 389 }() 390 391 Config.SlowSearchLogThreshold = 1 * time.Minute 392 393 query := NewTermQuery("water") 394 req := NewSearchRequest(query) 395 _, err = index.Search(req) 396 if err != nil { 397 t.Fatal(err) 398 } 399 400 if sdw.sawData { 401 t.Errorf("expected to not see slow query logged, but did") 402 } 403 404 sq := &slowQuery{ 405 actual: query, 406 delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms 407 } 408 req.Query = sq 409 Config.SlowSearchLogThreshold = 1 * time.Microsecond 410 _, err = index.Search(req) 411 if err != nil { 412 t.Fatal(err) 413 } 414 415 if !sdw.sawData { 416 t.Errorf("expected to see slow query logged, but didn't") 417 } 418} 419 420type sawDataWriter struct { 421 sawData bool 422} 423 424func (s *sawDataWriter) Write(p []byte) (n int, err error) { 425 s.sawData = true 426 return len(p), nil 427} 428 429func TestStoredFieldPreserved(t *testing.T) { 430 defer func() { 431 err := os.RemoveAll("testidx") 432 if err != nil { 433 t.Fatal(err) 434 } 435 }() 436 437 index, err := New("testidx", NewIndexMapping()) 438 if err != nil { 439 t.Fatal(err) 440 } 441 defer func() { 442 err := index.Close() 443 if err != nil { 444 t.Fatal(err) 445 } 446 }() 447 448 doca := map[string]interface{}{ 449 "name": "Marty", 450 "desc": "GopherCON India", 451 "bool": true, 452 "num": float64(1), 453 } 454 err = index.Index("a", doca) 455 if err != nil { 456 t.Error(err) 457 } 458 459 q := NewTermQuery("marty") 460 req := NewSearchRequest(q) 461 req.Fields = []string{"name", "desc", "bool", "num"} 462 res, err := index.Search(req) 463 if err != nil { 464 t.Error(err) 465 } 466 467 if len(res.Hits) != 1 { 468 t.Fatalf("expected 1 hit, got %d", len(res.Hits)) 469 } 470 if res.Hits[0].Fields["name"] != "Marty" { 471 t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"]) 472 } 473 if res.Hits[0].Fields["desc"] != "GopherCON India" { 474 t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"]) 475 } 476 if res.Hits[0].Fields["num"] != float64(1) { 477 t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"]) 478 } 479 if res.Hits[0].Fields["bool"] != true { 480 t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"]) 481 } 482} 483 484func TestDict(t *testing.T) { 485 defer func() { 486 err := os.RemoveAll("testidx") 487 if err != nil { 488 t.Fatal(err) 489 } 490 }() 491 492 index, err := New("testidx", NewIndexMapping()) 493 if err != nil { 494 t.Fatal(err) 495 } 496 497 doca := map[string]interface{}{ 498 "name": "marty", 499 "desc": "gophercon india", 500 } 501 err = index.Index("a", doca) 502 if err != nil { 503 t.Error(err) 504 } 505 506 docy := map[string]interface{}{ 507 "name": "jasper", 508 "desc": "clojure", 509 } 510 err = index.Index("y", docy) 511 if err != nil { 512 t.Error(err) 513 } 514 515 docx := map[string]interface{}{ 516 "name": "rose", 517 "desc": "googler", 518 } 519 err = index.Index("x", docx) 520 if err != nil { 521 t.Error(err) 522 } 523 524 dict, err := index.FieldDict("name") 525 if err != nil { 526 t.Error(err) 527 } 528 529 terms := []string{} 530 de, err := dict.Next() 531 for err == nil && de != nil { 532 terms = append(terms, string(de.Term)) 533 de, err = dict.Next() 534 } 535 536 expectedTerms := []string{"jasper", "marty", "rose"} 537 if !reflect.DeepEqual(terms, expectedTerms) { 538 t.Errorf("expected %v, got %v", expectedTerms, terms) 539 } 540 541 err = dict.Close() 542 if err != nil { 543 t.Fatal(err) 544 } 545 546 // test start and end range 547 dict, err = index.FieldDictRange("name", []byte("marty"), []byte("rose")) 548 if err != nil { 549 t.Error(err) 550 } 551 552 terms = []string{} 553 de, err = dict.Next() 554 for err == nil && de != nil { 555 terms = append(terms, string(de.Term)) 556 de, err = dict.Next() 557 } 558 559 expectedTerms = []string{"marty", "rose"} 560 if !reflect.DeepEqual(terms, expectedTerms) { 561 t.Errorf("expected %v, got %v", expectedTerms, terms) 562 } 563 564 err = dict.Close() 565 if err != nil { 566 t.Fatal(err) 567 } 568 569 docz := map[string]interface{}{ 570 "name": "prefix", 571 "desc": "bob cat cats catting dog doggy zoo", 572 } 573 err = index.Index("z", docz) 574 if err != nil { 575 t.Error(err) 576 } 577 578 dict, err = index.FieldDictPrefix("desc", []byte("cat")) 579 if err != nil { 580 t.Error(err) 581 } 582 583 terms = []string{} 584 de, err = dict.Next() 585 for err == nil && de != nil { 586 terms = append(terms, string(de.Term)) 587 de, err = dict.Next() 588 } 589 590 expectedTerms = []string{"cat", "cats", "catting"} 591 if !reflect.DeepEqual(terms, expectedTerms) { 592 t.Errorf("expected %v, got %v", expectedTerms, terms) 593 } 594 595 stats := index.Stats() 596 if stats == nil { 597 t.Errorf("expected IndexStat, got nil") 598 } 599 600 err = dict.Close() 601 if err != nil { 602 t.Fatal(err) 603 } 604 605 err = index.Close() 606 if err != nil { 607 t.Fatal(err) 608 } 609} 610 611func TestBatchString(t *testing.T) { 612 defer func() { 613 err := os.RemoveAll("testidx") 614 if err != nil { 615 t.Fatal(err) 616 } 617 }() 618 619 index, err := New("testidx", NewIndexMapping()) 620 if err != nil { 621 t.Fatal(err) 622 } 623 defer func() { 624 err := index.Close() 625 if err != nil { 626 t.Fatal(err) 627 } 628 }() 629 630 batch := index.NewBatch() 631 err = batch.Index("a", []byte("{}")) 632 if err != nil { 633 t.Fatal(err) 634 } 635 batch.Delete("b") 636 batch.SetInternal([]byte("c"), []byte{}) 637 batch.DeleteInternal([]byte("d")) 638 639 batchStr := batch.String() 640 if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") { 641 t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not") 642 } 643 if !strings.Contains(batchStr, "INDEX - 'a'") { 644 t.Errorf("expected to contain INDEX - 'a', did not") 645 } 646 if !strings.Contains(batchStr, "DELETE - 'b'") { 647 t.Errorf("expected to contain DELETE - 'b', did not") 648 } 649 if !strings.Contains(batchStr, "SET INTERNAL - 'c'") { 650 t.Errorf("expected to contain SET INTERNAL - 'c', did not") 651 } 652 if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") { 653 t.Errorf("expected to contain DELETE INTERNAL - 'd', did not") 654 } 655 656} 657 658func TestIndexMetadataRaceBug198(t *testing.T) { 659 defer func() { 660 err := os.RemoveAll("testidx") 661 if err != nil { 662 t.Fatal(err) 663 } 664 }() 665 666 index, err := New("testidx", NewIndexMapping()) 667 if err != nil { 668 t.Fatal(err) 669 } 670 defer func() { 671 err := index.Close() 672 if err != nil { 673 t.Fatal(err) 674 } 675 }() 676 677 wg := sync.WaitGroup{} 678 wg.Add(1) 679 done := make(chan struct{}) 680 go func() { 681 for { 682 select { 683 case <-done: 684 wg.Done() 685 return 686 default: 687 _, err2 := index.DocCount() 688 if err2 != nil { 689 t.Fatal(err2) 690 } 691 } 692 } 693 }() 694 695 for i := 0; i < 100; i++ { 696 batch := index.NewBatch() 697 err = batch.Index("a", []byte("{}")) 698 if err != nil { 699 t.Fatal(err) 700 } 701 err = index.Batch(batch) 702 if err != nil { 703 t.Fatal(err) 704 } 705 } 706 close(done) 707 wg.Wait() 708} 709 710func TestSortMatchSearch(t *testing.T) { 711 defer func() { 712 err := os.RemoveAll("testidx") 713 if err != nil { 714 t.Fatal(err) 715 } 716 }() 717 718 index, err := New("testidx", NewIndexMapping()) 719 if err != nil { 720 t.Fatal(err) 721 } 722 723 names := []string{"Noam", "Uri", "David", "Yosef", "Eitan", "Itay", "Ariel", "Daniel", "Omer", "Yogev", "Yehonatan", "Moshe", "Mohammed", "Yusuf", "Omar"} 724 days := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} 725 numbers := []string{"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten", "Eleven", "Twelve"} 726 for i := 0; i < 200; i++ { 727 doc := make(map[string]interface{}) 728 doc["Name"] = names[i%len(names)] 729 doc["Day"] = days[i%len(days)] 730 doc["Number"] = numbers[i%len(numbers)] 731 err = index.Index(fmt.Sprintf("%d", i), doc) 732 if err != nil { 733 t.Fatal(err) 734 } 735 } 736 737 req := NewSearchRequest(NewMatchQuery("One")) 738 req.SortBy([]string{"Day", "Name"}) 739 req.Fields = []string{"*"} 740 sr, err := index.Search(req) 741 if err != nil { 742 t.Fatal(err) 743 } 744 prev := "" 745 for _, hit := range sr.Hits { 746 val := hit.Fields["Day"].(string) 747 if prev > val { 748 t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val) 749 } 750 prev = val 751 } 752 err = index.Close() 753 if err != nil { 754 t.Fatal(err) 755 } 756} 757 758func TestIndexCountMatchSearch(t *testing.T) { 759 defer func() { 760 err := os.RemoveAll("testidx") 761 if err != nil { 762 t.Fatal(err) 763 } 764 }() 765 766 index, err := New("testidx", NewIndexMapping()) 767 if err != nil { 768 t.Fatal(err) 769 } 770 771 var wg sync.WaitGroup 772 for i := 0; i < 10; i++ { 773 wg.Add(1) 774 go func(i int) { 775 b := index.NewBatch() 776 for j := 0; j < 200; j++ { 777 id := fmt.Sprintf("%d", (i*200)+j) 778 doc := struct { 779 Body string 780 }{ 781 Body: "match", 782 } 783 err := b.Index(id, doc) 784 if err != nil { 785 t.Fatal(err) 786 } 787 } 788 err := index.Batch(b) 789 if err != nil { 790 t.Fatal(err) 791 } 792 wg.Done() 793 }(i) 794 } 795 wg.Wait() 796 797 // search for something that should match all documents 798 sr, err := index.Search(NewSearchRequest(NewMatchQuery("match"))) 799 if err != nil { 800 t.Fatal(err) 801 } 802 803 // get the index document count 804 dc, err := index.DocCount() 805 if err != nil { 806 t.Fatal(err) 807 } 808 809 // make sure test is working correctly, doc count should 2000 810 if dc != 2000 { 811 t.Errorf("expected doc count 2000, got %d", dc) 812 } 813 814 // make sure our search found all the documents 815 if dc != sr.Total { 816 t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc) 817 } 818 819 err = index.Close() 820 if err != nil { 821 t.Fatal(err) 822 } 823} 824 825func TestBatchReset(t *testing.T) { 826 defer func() { 827 err := os.RemoveAll("testidx") 828 if err != nil { 829 t.Fatal(err) 830 } 831 }() 832 833 index, err := New("testidx", NewIndexMapping()) 834 if err != nil { 835 t.Fatal(err) 836 } 837 838 batch := index.NewBatch() 839 err = batch.Index("k1", struct { 840 Body string 841 }{ 842 Body: "v1", 843 }) 844 if err != nil { 845 t.Error(err) 846 } 847 batch.Delete("k2") 848 batch.SetInternal([]byte("k3"), []byte("v3")) 849 batch.DeleteInternal([]byte("k4")) 850 851 if batch.Size() != 4 { 852 t.Logf("%v", batch) 853 t.Errorf("expected batch size 4, got %d", batch.Size()) 854 } 855 856 batch.Reset() 857 858 if batch.Size() != 0 { 859 t.Errorf("expected batch size 0 after reset, got %d", batch.Size()) 860 } 861 862 err = index.Close() 863 if err != nil { 864 t.Fatal(err) 865 } 866} 867 868func TestDocumentFieldArrayPositions(t *testing.T) { 869 defer func() { 870 err := os.RemoveAll("testidx") 871 if err != nil { 872 t.Fatal(err) 873 } 874 }() 875 876 index, err := New("testidx", NewIndexMapping()) 877 if err != nil { 878 t.Fatal(err) 879 } 880 881 // index a document with an array of strings 882 err = index.Index("k", struct { 883 Messages []string 884 }{ 885 Messages: []string{ 886 "first", 887 "second", 888 "third", 889 "last", 890 }, 891 }) 892 if err != nil { 893 t.Fatal(err) 894 } 895 896 // load the document 897 doc, err := index.Document("k") 898 if err != nil { 899 t.Fatal(err) 900 } 901 902 for _, f := range doc.Fields { 903 if reflect.DeepEqual(f.Value(), []byte("first")) { 904 ap := f.ArrayPositions() 905 if len(ap) < 1 { 906 t.Errorf("expected an array position, got none") 907 continue 908 } 909 if ap[0] != 0 { 910 t.Errorf("expected 'first' in array position 0, got %d", ap[0]) 911 } 912 } 913 if reflect.DeepEqual(f.Value(), []byte("second")) { 914 ap := f.ArrayPositions() 915 if len(ap) < 1 { 916 t.Errorf("expected an array position, got none") 917 continue 918 } 919 if ap[0] != 1 { 920 t.Errorf("expected 'second' in array position 1, got %d", ap[0]) 921 } 922 } 923 if reflect.DeepEqual(f.Value(), []byte("third")) { 924 ap := f.ArrayPositions() 925 if len(ap) < 1 { 926 t.Errorf("expected an array position, got none") 927 continue 928 } 929 if ap[0] != 2 { 930 t.Errorf("expected 'third' in array position 2, got %d", ap[0]) 931 } 932 } 933 if reflect.DeepEqual(f.Value(), []byte("last")) { 934 ap := f.ArrayPositions() 935 if len(ap) < 1 { 936 t.Errorf("expected an array position, got none") 937 continue 938 } 939 if ap[0] != 3 { 940 t.Errorf("expected 'last' in array position 3, got %d", ap[0]) 941 } 942 } 943 } 944 945 // now index a document in the same field with a single string 946 err = index.Index("k2", struct { 947 Messages string 948 }{ 949 Messages: "only", 950 }) 951 if err != nil { 952 t.Fatal(err) 953 } 954 955 // load the document 956 doc, err = index.Document("k2") 957 if err != nil { 958 t.Fatal(err) 959 } 960 961 for _, f := range doc.Fields { 962 if reflect.DeepEqual(f.Value(), []byte("only")) { 963 ap := f.ArrayPositions() 964 if len(ap) != 0 { 965 t.Errorf("expected no array positions, got %d", len(ap)) 966 continue 967 } 968 } 969 } 970 971 err = index.Close() 972 if err != nil { 973 t.Fatal(err) 974 } 975} 976 977func TestKeywordSearchBug207(t *testing.T) { 978 defer func() { 979 err := os.RemoveAll("testidx") 980 if err != nil { 981 t.Fatal(err) 982 } 983 }() 984 985 f := NewTextFieldMapping() 986 f.Analyzer = keyword.Name 987 988 m := NewIndexMapping() 989 m.DefaultMapping = NewDocumentMapping() 990 m.DefaultMapping.AddFieldMappingsAt("Body", f) 991 992 index, err := New("testidx", m) 993 if err != nil { 994 t.Fatal(err) 995 } 996 997 doc1 := struct { 998 Body string 999 }{ 1000 Body: "a555c3bb06f7a127cda000005", 1001 } 1002 1003 err = index.Index("a", doc1) 1004 if err != nil { 1005 t.Fatal(err) 1006 } 1007 1008 doc2 := struct { 1009 Body string 1010 }{ 1011 Body: "555c3bb06f7a127cda000005", 1012 } 1013 1014 err = index.Index("b", doc2) 1015 if err != nil { 1016 t.Fatal(err) 1017 } 1018 1019 // now search for these terms 1020 sreq := NewSearchRequest(NewTermQuery("a555c3bb06f7a127cda000005")) 1021 sres, err := index.Search(sreq) 1022 if err != nil { 1023 t.Fatal(err) 1024 } 1025 if sres.Total != 1 { 1026 t.Errorf("expected 1 result, got %d", sres.Total) 1027 } 1028 if sres.Hits[0].ID != "a" { 1029 t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID) 1030 } 1031 1032 sreq = NewSearchRequest(NewTermQuery("555c3bb06f7a127cda000005")) 1033 sres, err = index.Search(sreq) 1034 if err != nil { 1035 t.Fatal(err) 1036 } 1037 if sres.Total != 1 { 1038 t.Errorf("expected 1 result, got %d", sres.Total) 1039 } 1040 if sres.Hits[0].ID != "b" { 1041 t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID) 1042 } 1043 1044 // now do the same searches using query strings 1045 sreq = NewSearchRequest(NewQueryStringQuery("Body:a555c3bb06f7a127cda000005")) 1046 sres, err = index.Search(sreq) 1047 if err != nil { 1048 t.Fatal(err) 1049 } 1050 if sres.Total != 1 { 1051 t.Errorf("expected 1 result, got %d", sres.Total) 1052 } 1053 if sres.Hits[0].ID != "a" { 1054 t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID) 1055 } 1056 1057 sreq = NewSearchRequest(NewQueryStringQuery(`Body:555c3bb06f7a127cda000005`)) 1058 sres, err = index.Search(sreq) 1059 if err != nil { 1060 t.Fatal(err) 1061 } 1062 if sres.Total != 1 { 1063 t.Errorf("expected 1 result, got %d", sres.Total) 1064 } 1065 if sres.Hits[0].ID != "b" { 1066 t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID) 1067 } 1068 1069 err = index.Close() 1070 if err != nil { 1071 t.Fatal(err) 1072 } 1073} 1074 1075func TestTermVectorArrayPositions(t *testing.T) { 1076 defer func() { 1077 err := os.RemoveAll("testidx") 1078 if err != nil { 1079 t.Fatal(err) 1080 } 1081 }() 1082 1083 index, err := New("testidx", NewIndexMapping()) 1084 if err != nil { 1085 t.Fatal(err) 1086 } 1087 1088 // index a document with an array of strings 1089 err = index.Index("k", struct { 1090 Messages []string 1091 }{ 1092 Messages: []string{ 1093 "first", 1094 "second", 1095 "third", 1096 "last", 1097 }, 1098 }) 1099 if err != nil { 1100 t.Fatal(err) 1101 } 1102 1103 // search for this document in all field 1104 tq := NewTermQuery("second") 1105 tsr := NewSearchRequest(tq) 1106 tsr.IncludeLocations = true 1107 results, err := index.Search(tsr) 1108 if err != nil { 1109 t.Fatal(err) 1110 } 1111 if results.Total != 1 { 1112 t.Fatalf("expected 1 result, got %d", results.Total) 1113 } 1114 if len(results.Hits[0].Locations["Messages"]["second"]) < 1 { 1115 t.Fatalf("expected at least one location") 1116 } 1117 if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 { 1118 t.Fatalf("expected at least one location array position") 1119 } 1120 if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 { 1121 t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0]) 1122 } 1123 1124 // repeat search for this document in Messages field 1125 tq2 := NewTermQuery("third") 1126 tq2.SetField("Messages") 1127 tsr = NewSearchRequest(tq2) 1128 tsr.IncludeLocations = true 1129 results, err = index.Search(tsr) 1130 if err != nil { 1131 t.Fatal(err) 1132 } 1133 if results.Total != 1 { 1134 t.Fatalf("expected 1 result, got %d", results.Total) 1135 } 1136 if len(results.Hits[0].Locations["Messages"]["third"]) < 1 { 1137 t.Fatalf("expected at least one location") 1138 } 1139 if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 { 1140 t.Fatalf("expected at least one location array position") 1141 } 1142 if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 { 1143 t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0]) 1144 } 1145 1146 err = index.Close() 1147 if err != nil { 1148 t.Fatal(err) 1149 } 1150} 1151 1152func TestDocumentStaticMapping(t *testing.T) { 1153 defer func() { 1154 err := os.RemoveAll("testidx") 1155 if err != nil { 1156 t.Fatal(err) 1157 } 1158 }() 1159 1160 m := NewIndexMapping() 1161 m.DefaultMapping = NewDocumentStaticMapping() 1162 m.DefaultMapping.AddFieldMappingsAt("Text", NewTextFieldMapping()) 1163 m.DefaultMapping.AddFieldMappingsAt("Date", NewDateTimeFieldMapping()) 1164 m.DefaultMapping.AddFieldMappingsAt("Numeric", NewNumericFieldMapping()) 1165 1166 index, err := New("testidx", m) 1167 if err != nil { 1168 t.Fatal(err) 1169 } 1170 1171 doc1 := struct { 1172 Text string 1173 IgnoredText string 1174 Numeric float64 1175 IgnoredNumeric float64 1176 Date time.Time 1177 IgnoredDate time.Time 1178 }{ 1179 Text: "valid text", 1180 IgnoredText: "ignored text", 1181 Numeric: 10, 1182 IgnoredNumeric: 20, 1183 Date: time.Unix(1, 0), 1184 IgnoredDate: time.Unix(2, 0), 1185 } 1186 1187 err = index.Index("a", doc1) 1188 if err != nil { 1189 t.Fatal(err) 1190 } 1191 1192 fields, err := index.Fields() 1193 if err != nil { 1194 t.Fatal(err) 1195 } 1196 sort.Strings(fields) 1197 expectedFields := []string{"Date", "Numeric", "Text", "_all"} 1198 if len(fields) < len(expectedFields) { 1199 t.Fatalf("invalid field count: %d", len(fields)) 1200 } 1201 for i, expected := range expectedFields { 1202 if expected != fields[i] { 1203 t.Fatalf("unexpected field[%d]: %s", i, fields[i]) 1204 } 1205 } 1206 1207 err = index.Close() 1208 if err != nil { 1209 t.Fatal(err) 1210 } 1211} 1212 1213func TestIndexEmptyDocId(t *testing.T) { 1214 defer func() { 1215 err := os.RemoveAll("testidx") 1216 if err != nil { 1217 t.Fatal(err) 1218 } 1219 }() 1220 1221 index, err := New("testidx", NewIndexMapping()) 1222 if err != nil { 1223 t.Fatal(err) 1224 } 1225 defer func() { 1226 err := index.Close() 1227 if err != nil { 1228 t.Fatal(err) 1229 } 1230 }() 1231 1232 doc := map[string]interface{}{ 1233 "body": "nodocid", 1234 } 1235 1236 err = index.Index("", doc) 1237 if err != ErrorEmptyID { 1238 t.Errorf("expect index empty doc id to fail") 1239 } 1240 1241 err = index.Delete("") 1242 if err != ErrorEmptyID { 1243 t.Errorf("expect delete empty doc id to fail") 1244 } 1245 1246 batch := index.NewBatch() 1247 err = batch.Index("", doc) 1248 if err != ErrorEmptyID { 1249 t.Errorf("expect index empty doc id in batch to fail") 1250 } 1251 1252 batch.Delete("") 1253 if batch.Size() > 0 { 1254 t.Errorf("expect delete empty doc id in batch to be ignored") 1255 } 1256} 1257 1258func TestDateTimeFieldMappingIssue287(t *testing.T) { 1259 defer func() { 1260 err := os.RemoveAll("testidx") 1261 if err != nil { 1262 t.Fatal(err) 1263 } 1264 }() 1265 1266 f := NewDateTimeFieldMapping() 1267 1268 m := NewIndexMapping() 1269 m.DefaultMapping = NewDocumentMapping() 1270 m.DefaultMapping.AddFieldMappingsAt("Date", f) 1271 1272 index, err := New("testidx", m) 1273 if err != nil { 1274 t.Fatal(err) 1275 } 1276 1277 type doc struct { 1278 Date time.Time 1279 } 1280 1281 now := time.Now() 1282 1283 // 3hr ago to 1hr ago 1284 for i := 0; i < 3; i++ { 1285 d := doc{now.Add(time.Duration((i - 3)) * time.Hour)} 1286 1287 err = index.Index(strconv.FormatInt(int64(i), 10), d) 1288 if err != nil { 1289 t.Fatal(err) 1290 } 1291 } 1292 1293 // search range across all docs 1294 start := now.Add(-4 * time.Hour) 1295 end := now 1296 sreq := NewSearchRequest(NewDateRangeQuery(start, end)) 1297 sres, err := index.Search(sreq) 1298 if err != nil { 1299 t.Fatal(err) 1300 } 1301 if sres.Total != 3 { 1302 t.Errorf("expected 3 results, got %d", sres.Total) 1303 } 1304 1305 // search range includes only oldest 1306 start = now.Add(-4 * time.Hour) 1307 end = now.Add(-121 * time.Minute) 1308 sreq = NewSearchRequest(NewDateRangeQuery(start, end)) 1309 sres, err = index.Search(sreq) 1310 if err != nil { 1311 t.Fatal(err) 1312 } 1313 if sres.Total != 1 { 1314 t.Errorf("expected 1 results, got %d", sres.Total) 1315 } 1316 if sres.Total > 0 && sres.Hits[0].ID != "0" { 1317 t.Errorf("expecated id '0', got '%s'", sres.Hits[0].ID) 1318 } 1319 1320 // search range includes only newest 1321 start = now.Add(-61 * time.Minute) 1322 end = now 1323 sreq = NewSearchRequest(NewDateRangeQuery(start, end)) 1324 sres, err = index.Search(sreq) 1325 if err != nil { 1326 t.Fatal(err) 1327 } 1328 if sres.Total != 1 { 1329 t.Errorf("expected 1 results, got %d", sres.Total) 1330 } 1331 if sres.Total > 0 && sres.Hits[0].ID != "2" { 1332 t.Errorf("expecated id '2', got '%s'", sres.Hits[0].ID) 1333 } 1334 1335 err = index.Close() 1336 if err != nil { 1337 t.Fatal(err) 1338 } 1339} 1340 1341func TestDocumentFieldArrayPositionsBug295(t *testing.T) { 1342 defer func() { 1343 err := os.RemoveAll("testidx") 1344 if err != nil { 1345 t.Fatal(err) 1346 } 1347 }() 1348 1349 index, err := New("testidx", NewIndexMapping()) 1350 if err != nil { 1351 t.Fatal(err) 1352 } 1353 1354 // index a document with an array of strings 1355 err = index.Index("k", struct { 1356 Messages []string 1357 Another string 1358 MoreData []string 1359 }{ 1360 Messages: []string{ 1361 "bleve", 1362 "bleve", 1363 }, 1364 Another: "text", 1365 MoreData: []string{ 1366 "a", 1367 "b", 1368 "c", 1369 "bleve", 1370 }, 1371 }) 1372 if err != nil { 1373 t.Fatal(err) 1374 } 1375 1376 // search for it in the messages field 1377 tq := NewTermQuery("bleve") 1378 tq.SetField("Messages") 1379 tsr := NewSearchRequest(tq) 1380 tsr.IncludeLocations = true 1381 results, err := index.Search(tsr) 1382 if err != nil { 1383 t.Fatal(err) 1384 } 1385 if results.Total != 1 { 1386 t.Fatalf("expected 1 result, got %d", results.Total) 1387 } 1388 if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { 1389 t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) 1390 } 1391 if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { 1392 t.Errorf("expected array position to be 0") 1393 } 1394 if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { 1395 t.Errorf("expected array position to be 1") 1396 } 1397 1398 // search for it in all 1399 tq = NewTermQuery("bleve") 1400 tsr = NewSearchRequest(tq) 1401 tsr.IncludeLocations = true 1402 results, err = index.Search(tsr) 1403 if err != nil { 1404 t.Fatal(err) 1405 } 1406 if results.Total != 1 { 1407 t.Fatalf("expected 1 result, got %d", results.Total) 1408 } 1409 if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { 1410 t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) 1411 } 1412 if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { 1413 t.Errorf("expected array position to be 0") 1414 } 1415 if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { 1416 t.Errorf("expected array position to be 1") 1417 } 1418 1419 err = index.Close() 1420 if err != nil { 1421 t.Fatal(err) 1422 } 1423} 1424 1425func TestBooleanFieldMappingIssue109(t *testing.T) { 1426 defer func() { 1427 err := os.RemoveAll("testidx") 1428 if err != nil { 1429 t.Fatal(err) 1430 } 1431 }() 1432 1433 m := NewIndexMapping() 1434 m.DefaultMapping = NewDocumentMapping() 1435 m.DefaultMapping.AddFieldMappingsAt("Bool", NewBooleanFieldMapping()) 1436 1437 index, err := New("testidx", m) 1438 if err != nil { 1439 t.Fatal(err) 1440 } 1441 1442 type doc struct { 1443 Bool bool 1444 } 1445 err = index.Index("true", &doc{Bool: true}) 1446 if err != nil { 1447 t.Fatal(err) 1448 } 1449 err = index.Index("false", &doc{Bool: false}) 1450 if err != nil { 1451 t.Fatal(err) 1452 } 1453 1454 q := NewBoolFieldQuery(true) 1455 q.SetField("Bool") 1456 sreq := NewSearchRequest(q) 1457 sres, err := index.Search(sreq) 1458 if err != nil { 1459 t.Fatal(err) 1460 } 1461 if sres.Total != 1 { 1462 t.Errorf("expected 1 results, got %d", sres.Total) 1463 } 1464 1465 q = NewBoolFieldQuery(false) 1466 q.SetField("Bool") 1467 sreq = NewSearchRequest(q) 1468 sres, err = index.Search(sreq) 1469 if err != nil { 1470 t.Fatal(err) 1471 } 1472 if sres.Total != 1 { 1473 t.Errorf("expected 1 results, got %d", sres.Total) 1474 } 1475 1476 sreq = NewSearchRequest(NewBoolFieldQuery(true)) 1477 sres, err = index.Search(sreq) 1478 if err != nil { 1479 t.Fatal(err) 1480 } 1481 if sres.Total != 1 { 1482 t.Errorf("expected 1 results, got %d", sres.Total) 1483 } 1484 1485 err = index.Close() 1486 if err != nil { 1487 t.Fatal(err) 1488 } 1489} 1490 1491func TestSearchTimeout(t *testing.T) { 1492 defer func() { 1493 err := os.RemoveAll("testidx") 1494 if err != nil { 1495 t.Fatal(err) 1496 } 1497 }() 1498 1499 index, err := New("testidx", NewIndexMapping()) 1500 if err != nil { 1501 t.Fatal(err) 1502 } 1503 defer func() { 1504 err := index.Close() 1505 if err != nil { 1506 t.Fatal(err) 1507 } 1508 }() 1509 1510 // first run a search with an absurdly long timeout (should succeeed) 1511 ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) 1512 query := NewTermQuery("water") 1513 req := NewSearchRequest(query) 1514 _, err = index.SearchInContext(ctx, req) 1515 if err != nil { 1516 t.Fatal(err) 1517 } 1518 1519 // now run a search again with an absurdly low timeout (should timeout) 1520 ctx, _ = context.WithTimeout(context.Background(), 1*time.Microsecond) 1521 sq := &slowQuery{ 1522 actual: query, 1523 delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms 1524 } 1525 req.Query = sq 1526 _, err = index.SearchInContext(ctx, req) 1527 if err != context.DeadlineExceeded { 1528 t.Fatalf("exected %v, got: %v", context.DeadlineExceeded, err) 1529 } 1530 1531 // now run a search with a long timeout, but with a long query, and cancel it 1532 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 1533 sq = &slowQuery{ 1534 actual: query, 1535 delay: 100 * time.Millisecond, // on Windows timer resolution is 15ms 1536 } 1537 req = NewSearchRequest(sq) 1538 cancel() 1539 _, err = index.SearchInContext(ctx, req) 1540 if err != context.Canceled { 1541 t.Fatalf("exected %v, got: %v", context.Canceled, err) 1542 } 1543} 1544 1545// TestConfigCache exposes a concurrent map write with go 1.6 1546func TestConfigCache(t *testing.T) { 1547 for i := 0; i < 100; i++ { 1548 go func() { 1549 _, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter) 1550 if err != nil { 1551 t.Error(err) 1552 } 1553 }() 1554 } 1555} 1556 1557func TestBatchRaceBug260(t *testing.T) { 1558 defer func() { 1559 err := os.RemoveAll("testidx") 1560 if err != nil { 1561 t.Fatal(err) 1562 } 1563 }() 1564 i, err := New("testidx", NewIndexMapping()) 1565 if err != nil { 1566 t.Fatal(err) 1567 } 1568 defer func() { 1569 err := i.Close() 1570 if err != nil { 1571 t.Fatal(err) 1572 } 1573 }() 1574 b := i.NewBatch() 1575 err = b.Index("1", 1) 1576 if err != nil { 1577 t.Fatal(err) 1578 } 1579 err = i.Batch(b) 1580 if err != nil { 1581 t.Fatal(err) 1582 } 1583 b.Reset() 1584 err = b.Index("2", 2) 1585 if err != nil { 1586 t.Fatal(err) 1587 } 1588 err = i.Batch(b) 1589 if err != nil { 1590 t.Fatal(err) 1591 } 1592 b.Reset() 1593} 1594 1595func BenchmarkBatchOverhead(b *testing.B) { 1596 defer func() { 1597 err := os.RemoveAll("testidx") 1598 if err != nil { 1599 b.Fatal(err) 1600 } 1601 }() 1602 m := NewIndexMapping() 1603 i, err := NewUsing("testidx", m, Config.DefaultIndexType, null.Name, nil) 1604 if err != nil { 1605 b.Fatal(err) 1606 } 1607 for n := 0; n < b.N; n++ { 1608 // put 1000 items in a batch 1609 batch := i.NewBatch() 1610 for i := 0; i < 1000; i++ { 1611 err = batch.Index(fmt.Sprintf("%d", i), map[string]interface{}{"name": "bleve"}) 1612 if err != nil { 1613 b.Fatal(err) 1614 } 1615 } 1616 err = i.Batch(batch) 1617 if err != nil { 1618 b.Fatal(err) 1619 } 1620 batch.Reset() 1621 } 1622} 1623 1624func TestOpenReadonlyMultiple(t *testing.T) { 1625 defer func() { 1626 err := os.RemoveAll("testidx") 1627 if err != nil { 1628 t.Fatal(err) 1629 } 1630 }() 1631 1632 // build an index and close it 1633 index, err := New("testidx", NewIndexMapping()) 1634 if err != nil { 1635 t.Fatal(err) 1636 } 1637 1638 doca := map[string]interface{}{ 1639 "name": "marty", 1640 "desc": "gophercon india", 1641 } 1642 err = index.Index("a", doca) 1643 if err != nil { 1644 t.Fatal(err) 1645 } 1646 1647 err = index.Close() 1648 if err != nil { 1649 t.Fatal(err) 1650 } 1651 1652 // now open it read-only 1653 index, err = OpenUsing("testidx", map[string]interface{}{ 1654 "read_only": true, 1655 }) 1656 1657 if err != nil { 1658 t.Fatal(err) 1659 } 1660 1661 // now open it again 1662 index2, err := OpenUsing("testidx", map[string]interface{}{ 1663 "read_only": true, 1664 }) 1665 1666 if err != nil { 1667 t.Fatal(err) 1668 } 1669 1670 err = index.Close() 1671 if err != nil { 1672 t.Fatal(err) 1673 } 1674 err = index2.Close() 1675 if err != nil { 1676 t.Fatal(err) 1677 } 1678} 1679 1680// TestBug408 tests for VERY large values of size, even though actual result 1681// set may be reasonable size 1682func TestBug408(t *testing.T) { 1683 type TestStruct struct { 1684 ID string `json:"id"` 1685 UserID *string `json:"user_id"` 1686 } 1687 1688 docMapping := NewDocumentMapping() 1689 docMapping.AddFieldMappingsAt("id", NewTextFieldMapping()) 1690 docMapping.AddFieldMappingsAt("user_id", NewTextFieldMapping()) 1691 1692 indexMapping := NewIndexMapping() 1693 indexMapping.DefaultMapping = docMapping 1694 1695 index, err := NewMemOnly(indexMapping) 1696 if err != nil { 1697 t.Fatal(err) 1698 } 1699 1700 numToTest := 10 1701 matchUserID := "match" 1702 noMatchUserID := "no_match" 1703 matchingDocIds := make(map[string]struct{}) 1704 1705 for i := 0; i < numToTest; i++ { 1706 ds := &TestStruct{"id_" + strconv.Itoa(i), nil} 1707 if i%2 == 0 { 1708 ds.UserID = &noMatchUserID 1709 } else { 1710 ds.UserID = &matchUserID 1711 matchingDocIds[ds.ID] = struct{}{} 1712 } 1713 err = index.Index(ds.ID, ds) 1714 if err != nil { 1715 t.Fatal(err) 1716 } 1717 } 1718 1719 cnt, err := index.DocCount() 1720 if err != nil { 1721 t.Fatal(err) 1722 } 1723 if int(cnt) != numToTest { 1724 t.Fatalf("expected %d documents in index, got %d", numToTest, cnt) 1725 } 1726 1727 q := NewTermQuery(matchUserID) 1728 q.SetField("user_id") 1729 searchReq := NewSearchRequestOptions(q, math.MaxInt32, 0, false) 1730 results, err := index.Search(searchReq) 1731 if err != nil { 1732 t.Fatal(err) 1733 } 1734 if int(results.Total) != numToTest/2 { 1735 t.Fatalf("expected %d search hits, got %d", numToTest/2, results.Total) 1736 } 1737 1738 for _, result := range results.Hits { 1739 if _, found := matchingDocIds[result.ID]; !found { 1740 t.Fatalf("document with ID %s not in results as expected", result.ID) 1741 } 1742 } 1743} 1744 1745func TestIndexAdvancedCountMatchSearch(t *testing.T) { 1746 defer func() { 1747 err := os.RemoveAll("testidx") 1748 if err != nil { 1749 t.Fatal(err) 1750 } 1751 }() 1752 1753 index, err := New("testidx", NewIndexMapping()) 1754 if err != nil { 1755 t.Fatal(err) 1756 } 1757 1758 var wg sync.WaitGroup 1759 for i := 0; i < 10; i++ { 1760 wg.Add(1) 1761 go func(i int) { 1762 b := index.NewBatch() 1763 for j := 0; j < 200; j++ { 1764 id := fmt.Sprintf("%d", (i*200)+j) 1765 1766 doc := &document.Document{ 1767 ID: id, 1768 Fields: []document.Field{ 1769 document.NewTextField("body", []uint64{}, []byte("match")), 1770 }, 1771 CompositeFields: []*document.CompositeField{ 1772 document.NewCompositeField("_all", true, []string{}, []string{}), 1773 }, 1774 } 1775 1776 err := b.IndexAdvanced(doc) 1777 if err != nil { 1778 t.Fatal(err) 1779 } 1780 } 1781 err := index.Batch(b) 1782 if err != nil { 1783 t.Fatal(err) 1784 } 1785 wg.Done() 1786 }(i) 1787 } 1788 wg.Wait() 1789 1790 // search for something that should match all documents 1791 sr, err := index.Search(NewSearchRequest(NewMatchQuery("match"))) 1792 if err != nil { 1793 t.Fatal(err) 1794 } 1795 1796 // get the index document count 1797 dc, err := index.DocCount() 1798 if err != nil { 1799 t.Fatal(err) 1800 } 1801 1802 // make sure test is working correctly, doc count should 2000 1803 if dc != 2000 { 1804 t.Errorf("expected doc count 2000, got %d", dc) 1805 } 1806 1807 // make sure our search found all the documents 1808 if dc != sr.Total { 1809 t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc) 1810 } 1811 1812 err = index.Close() 1813 if err != nil { 1814 t.Fatal(err) 1815 } 1816} 1817