1// Copyright 2012-present Oliver Eilhard. All rights reserved. 2// Use of this source code is governed by a MIT-license. 3// See http://olivere.mit-license.org/license.txt for details. 4 5package elastic 6 7import ( 8 "context" 9 "encoding/json" 10 "fmt" 11 "net/url" 12 "reflect" 13 "strings" 14 15 "github.com/olivere/elastic/uritemplates" 16) 17 18// Search for documents in Elasticsearch. 19type SearchService struct { 20 client *Client 21 searchSource *SearchSource 22 source interface{} 23 pretty bool 24 filterPath []string 25 searchType string 26 index []string 27 typ []string 28 routing string 29 preference string 30 requestCache *bool 31 ignoreUnavailable *bool 32 allowNoIndices *bool 33 expandWildcards string 34 maxResponseSize int64 35} 36 37// NewSearchService creates a new service for searching in Elasticsearch. 38func NewSearchService(client *Client) *SearchService { 39 builder := &SearchService{ 40 client: client, 41 searchSource: NewSearchSource(), 42 } 43 return builder 44} 45 46// SearchSource sets the search source builder to use with this service. 47func (s *SearchService) SearchSource(searchSource *SearchSource) *SearchService { 48 s.searchSource = searchSource 49 if s.searchSource == nil { 50 s.searchSource = NewSearchSource() 51 } 52 return s 53} 54 55// Source allows the user to set the request body manually without using 56// any of the structs and interfaces in Elastic. 57func (s *SearchService) Source(source interface{}) *SearchService { 58 s.source = source 59 return s 60} 61 62// FilterPath allows reducing the response, a mechanism known as 63// response filtering and described here: 64// https://www.elastic.co/guide/en/elasticsearch/reference/6.7/common-options.html#common-options-response-filtering. 65func (s *SearchService) FilterPath(filterPath ...string) *SearchService { 66 s.filterPath = append(s.filterPath, filterPath...) 67 return s 68} 69 70// Index sets the names of the indices to use for search. 71func (s *SearchService) Index(index ...string) *SearchService { 72 s.index = append(s.index, index...) 73 return s 74} 75 76// Types adds search restrictions for a list of types. 77func (s *SearchService) Type(typ ...string) *SearchService { 78 s.typ = append(s.typ, typ...) 79 return s 80} 81 82// Pretty enables the caller to indent the JSON output. 83func (s *SearchService) Pretty(pretty bool) *SearchService { 84 s.pretty = pretty 85 return s 86} 87 88// Timeout sets the timeout to use, e.g. "1s" or "1000ms". 89func (s *SearchService) Timeout(timeout string) *SearchService { 90 s.searchSource = s.searchSource.Timeout(timeout) 91 return s 92} 93 94// Profile sets the Profile API flag on the search source. 95// When enabled, a search executed by this service will return query 96// profiling data. 97func (s *SearchService) Profile(profile bool) *SearchService { 98 s.searchSource = s.searchSource.Profile(profile) 99 return s 100} 101 102// Collapse adds field collapsing. 103func (s *SearchService) Collapse(collapse *CollapseBuilder) *SearchService { 104 s.searchSource = s.searchSource.Collapse(collapse) 105 return s 106} 107 108// TimeoutInMillis sets the timeout in milliseconds. 109func (s *SearchService) TimeoutInMillis(timeoutInMillis int) *SearchService { 110 s.searchSource = s.searchSource.TimeoutInMillis(timeoutInMillis) 111 return s 112} 113 114// TerminateAfter specifies the maximum number of documents to collect for 115// each shard, upon reaching which the query execution will terminate early. 116func (s *SearchService) TerminateAfter(terminateAfter int) *SearchService { 117 s.searchSource = s.searchSource.TerminateAfter(terminateAfter) 118 return s 119} 120 121// SearchType sets the search operation type. Valid values are: 122// "dfs_query_then_fetch" and "query_then_fetch". 123// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-request-search-type.html 124// for details. 125func (s *SearchService) SearchType(searchType string) *SearchService { 126 s.searchType = searchType 127 return s 128} 129 130// Routing is a list of specific routing values to control the shards 131// the search will be executed on. 132func (s *SearchService) Routing(routings ...string) *SearchService { 133 s.routing = strings.Join(routings, ",") 134 return s 135} 136 137// Preference sets the preference to execute the search. Defaults to 138// randomize across shards ("random"). Can be set to "_local" to prefer 139// local shards, "_primary" to execute on primary shards only, 140// or a custom value which guarantees that the same order will be used 141// across different requests. 142func (s *SearchService) Preference(preference string) *SearchService { 143 s.preference = preference 144 return s 145} 146 147// RequestCache indicates whether the cache should be used for this 148// request or not, defaults to index level setting. 149func (s *SearchService) RequestCache(requestCache bool) *SearchService { 150 s.requestCache = &requestCache 151 return s 152} 153 154// Query sets the query to perform, e.g. MatchAllQuery. 155func (s *SearchService) Query(query Query) *SearchService { 156 s.searchSource = s.searchSource.Query(query) 157 return s 158} 159 160// PostFilter will be executed after the query has been executed and 161// only affects the search hits, not the aggregations. 162// This filter is always executed as the last filtering mechanism. 163func (s *SearchService) PostFilter(postFilter Query) *SearchService { 164 s.searchSource = s.searchSource.PostFilter(postFilter) 165 return s 166} 167 168// FetchSource indicates whether the response should contain the stored 169// _source for every hit. 170func (s *SearchService) FetchSource(fetchSource bool) *SearchService { 171 s.searchSource = s.searchSource.FetchSource(fetchSource) 172 return s 173} 174 175// FetchSourceContext indicates how the _source should be fetched. 176func (s *SearchService) FetchSourceContext(fetchSourceContext *FetchSourceContext) *SearchService { 177 s.searchSource = s.searchSource.FetchSourceContext(fetchSourceContext) 178 return s 179} 180 181// Highlight adds highlighting to the search. 182func (s *SearchService) Highlight(highlight *Highlight) *SearchService { 183 s.searchSource = s.searchSource.Highlight(highlight) 184 return s 185} 186 187// GlobalSuggestText defines the global text to use with all suggesters. 188// This avoids repetition. 189func (s *SearchService) GlobalSuggestText(globalText string) *SearchService { 190 s.searchSource = s.searchSource.GlobalSuggestText(globalText) 191 return s 192} 193 194// Suggester adds a suggester to the search. 195func (s *SearchService) Suggester(suggester Suggester) *SearchService { 196 s.searchSource = s.searchSource.Suggester(suggester) 197 return s 198} 199 200// Aggregation adds an aggreation to perform as part of the search. 201func (s *SearchService) Aggregation(name string, aggregation Aggregation) *SearchService { 202 s.searchSource = s.searchSource.Aggregation(name, aggregation) 203 return s 204} 205 206// MinScore sets the minimum score below which docs will be filtered out. 207func (s *SearchService) MinScore(minScore float64) *SearchService { 208 s.searchSource = s.searchSource.MinScore(minScore) 209 return s 210} 211 212// From index to start the search from. Defaults to 0. 213func (s *SearchService) From(from int) *SearchService { 214 s.searchSource = s.searchSource.From(from) 215 return s 216} 217 218// Size is the number of search hits to return. Defaults to 10. 219func (s *SearchService) Size(size int) *SearchService { 220 s.searchSource = s.searchSource.Size(size) 221 return s 222} 223 224// Explain indicates whether each search hit should be returned with 225// an explanation of the hit (ranking). 226func (s *SearchService) Explain(explain bool) *SearchService { 227 s.searchSource = s.searchSource.Explain(explain) 228 return s 229} 230 231// Version indicates whether each search hit should be returned with 232// a version associated to it. 233func (s *SearchService) Version(version bool) *SearchService { 234 s.searchSource = s.searchSource.Version(version) 235 return s 236} 237 238// Sort adds a sort order. 239func (s *SearchService) Sort(field string, ascending bool) *SearchService { 240 s.searchSource = s.searchSource.Sort(field, ascending) 241 return s 242} 243 244// SortWithInfo adds a sort order. 245func (s *SearchService) SortWithInfo(info SortInfo) *SearchService { 246 s.searchSource = s.searchSource.SortWithInfo(info) 247 return s 248} 249 250// SortBy adds a sort order. 251func (s *SearchService) SortBy(sorter ...Sorter) *SearchService { 252 s.searchSource = s.searchSource.SortBy(sorter...) 253 return s 254} 255 256// DocvalueField adds a single field to load from the field data cache 257// and return as part of the search. 258func (s *SearchService) DocvalueField(docvalueField string) *SearchService { 259 s.searchSource = s.searchSource.DocvalueField(docvalueField) 260 return s 261} 262 263// DocvalueFieldWithFormat adds a single field to load from the field data cache 264// and return as part of the search. 265func (s *SearchService) DocvalueFieldWithFormat(docvalueField DocvalueField) *SearchService { 266 s.searchSource = s.searchSource.DocvalueFieldWithFormat(docvalueField) 267 return s 268} 269 270// DocvalueFields adds one or more fields to load from the field data cache 271// and return as part of the search. 272func (s *SearchService) DocvalueFields(docvalueFields ...string) *SearchService { 273 s.searchSource = s.searchSource.DocvalueFields(docvalueFields...) 274 return s 275} 276 277// DocvalueFieldsWithFormat adds one or more fields to load from the field data cache 278// and return as part of the search. 279func (s *SearchService) DocvalueFieldsWithFormat(docvalueFields ...DocvalueField) *SearchService { 280 s.searchSource = s.searchSource.DocvalueFieldsWithFormat(docvalueFields...) 281 return s 282} 283 284// NoStoredFields indicates that no stored fields should be loaded, resulting in only 285// id and type to be returned per field. 286func (s *SearchService) NoStoredFields() *SearchService { 287 s.searchSource = s.searchSource.NoStoredFields() 288 return s 289} 290 291// StoredField adds a single field to load and return (note, must be stored) as 292// part of the search request. If none are specified, the source of the 293// document will be returned. 294func (s *SearchService) StoredField(fieldName string) *SearchService { 295 s.searchSource = s.searchSource.StoredField(fieldName) 296 return s 297} 298 299// StoredFields sets the fields to load and return as part of the search request. 300// If none are specified, the source of the document will be returned. 301func (s *SearchService) StoredFields(fields ...string) *SearchService { 302 s.searchSource = s.searchSource.StoredFields(fields...) 303 return s 304} 305 306// TrackScores is applied when sorting and controls if scores will be 307// tracked as well. Defaults to false. 308func (s *SearchService) TrackScores(trackScores bool) *SearchService { 309 s.searchSource = s.searchSource.TrackScores(trackScores) 310 return s 311} 312 313// SearchAfter allows a different form of pagination by using a live cursor, 314// using the results of the previous page to help the retrieval of the next. 315// 316// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-request-search-after.html 317func (s *SearchService) SearchAfter(sortValues ...interface{}) *SearchService { 318 s.searchSource = s.searchSource.SearchAfter(sortValues...) 319 return s 320} 321 322// IgnoreUnavailable indicates whether the specified concrete indices 323// should be ignored when unavailable (missing or closed). 324func (s *SearchService) IgnoreUnavailable(ignoreUnavailable bool) *SearchService { 325 s.ignoreUnavailable = &ignoreUnavailable 326 return s 327} 328 329// AllowNoIndices indicates whether to ignore if a wildcard indices 330// expression resolves into no concrete indices. (This includes `_all` string 331// or when no indices have been specified). 332func (s *SearchService) AllowNoIndices(allowNoIndices bool) *SearchService { 333 s.allowNoIndices = &allowNoIndices 334 return s 335} 336 337// ExpandWildcards indicates whether to expand wildcard expression to 338// concrete indices that are open, closed or both. 339func (s *SearchService) ExpandWildcards(expandWildcards string) *SearchService { 340 s.expandWildcards = expandWildcards 341 return s 342} 343 344// MaxResponseSize sets an upper limit on the response body size that we accept, 345// to guard against OOM situations. 346func (s *SearchService) MaxResponseSize(maxResponseSize int64) *SearchService { 347 s.maxResponseSize = maxResponseSize 348 return s 349} 350 351// buildURL builds the URL for the operation. 352func (s *SearchService) buildURL() (string, url.Values, error) { 353 var err error 354 var path string 355 356 if len(s.index) > 0 && len(s.typ) > 0 { 357 path, err = uritemplates.Expand("/{index}/{type}/_search", map[string]string{ 358 "index": strings.Join(s.index, ","), 359 "type": strings.Join(s.typ, ","), 360 }) 361 } else if len(s.index) > 0 { 362 path, err = uritemplates.Expand("/{index}/_search", map[string]string{ 363 "index": strings.Join(s.index, ","), 364 }) 365 } else if len(s.typ) > 0 { 366 path, err = uritemplates.Expand("/_all/{type}/_search", map[string]string{ 367 "type": strings.Join(s.typ, ","), 368 }) 369 } else { 370 path = "/_search" 371 } 372 if err != nil { 373 return "", url.Values{}, err 374 } 375 376 // Add query string parameters 377 params := url.Values{} 378 if s.pretty { 379 params.Set("pretty", fmt.Sprintf("%v", s.pretty)) 380 } 381 if s.searchType != "" { 382 params.Set("search_type", s.searchType) 383 } 384 if s.routing != "" { 385 params.Set("routing", s.routing) 386 } 387 if s.preference != "" { 388 params.Set("preference", s.preference) 389 } 390 if s.requestCache != nil { 391 params.Set("request_cache", fmt.Sprintf("%v", *s.requestCache)) 392 } 393 if s.allowNoIndices != nil { 394 params.Set("allow_no_indices", fmt.Sprintf("%v", *s.allowNoIndices)) 395 } 396 if s.expandWildcards != "" { 397 params.Set("expand_wildcards", s.expandWildcards) 398 } 399 if s.ignoreUnavailable != nil { 400 params.Set("ignore_unavailable", fmt.Sprintf("%v", *s.ignoreUnavailable)) 401 } 402 if len(s.filterPath) > 0 { 403 params.Set("filter_path", strings.Join(s.filterPath, ",")) 404 } 405 return path, params, nil 406} 407 408// Validate checks if the operation is valid. 409func (s *SearchService) Validate() error { 410 return nil 411} 412 413// Do executes the search and returns a SearchResult. 414func (s *SearchService) Do(ctx context.Context) (*SearchResult, error) { 415 // Check pre-conditions 416 if err := s.Validate(); err != nil { 417 return nil, err 418 } 419 420 // Get URL for request 421 path, params, err := s.buildURL() 422 if err != nil { 423 return nil, err 424 } 425 426 // Perform request 427 var body interface{} 428 if s.source != nil { 429 body = s.source 430 } else { 431 src, err := s.searchSource.Source() 432 if err != nil { 433 return nil, err 434 } 435 body = src 436 } 437 res, err := s.client.PerformRequest(ctx, PerformRequestOptions{ 438 Method: "POST", 439 Path: path, 440 Params: params, 441 Body: body, 442 MaxResponseSize: s.maxResponseSize, 443 }) 444 if err != nil { 445 return nil, err 446 } 447 448 // Return search results 449 ret := new(SearchResult) 450 if err := s.client.decoder.Decode(res.Body, ret); err != nil { 451 return nil, err 452 } 453 return ret, nil 454} 455 456// SearchResult is the result of a search in Elasticsearch. 457type SearchResult struct { 458 TookInMillis int64 `json:"took,omitempty"` // search time in milliseconds 459 ScrollId string `json:"_scroll_id,omitempty"` // only used with Scroll and Scan operations 460 Hits *SearchHits `json:"hits,omitempty"` // the actual search hits 461 Suggest SearchSuggest `json:"suggest,omitempty"` // results from suggesters 462 Aggregations Aggregations `json:"aggregations,omitempty"` // results from aggregations 463 TimedOut bool `json:"timed_out,omitempty"` // true if the search timed out 464 Error *ErrorDetails `json:"error,omitempty"` // only used in MultiGet 465 Profile *SearchProfile `json:"profile,omitempty"` // profiling results, if optional Profile API was active for this search 466 Shards *ShardsInfo `json:"_shards,omitempty"` // shard information 467} 468 469// TotalHits is a convenience function to return the number of hits for 470// a search result. 471func (r *SearchResult) TotalHits() int64 { 472 if r.Hits != nil { 473 return r.Hits.TotalHits 474 } 475 return 0 476} 477 478// Each is a utility function to iterate over all hits. It saves you from 479// checking for nil values. Notice that Each will ignore errors in 480// serializing JSON and hits with empty/nil _source will get an empty 481// value 482func (r *SearchResult) Each(typ reflect.Type) []interface{} { 483 if r.Hits == nil || r.Hits.Hits == nil || len(r.Hits.Hits) == 0 { 484 return nil 485 } 486 var slice []interface{} 487 for _, hit := range r.Hits.Hits { 488 v := reflect.New(typ).Elem() 489 if hit.Source == nil { 490 slice = append(slice, v.Interface()) 491 continue 492 } 493 if err := json.Unmarshal(*hit.Source, v.Addr().Interface()); err == nil { 494 slice = append(slice, v.Interface()) 495 } 496 } 497 return slice 498} 499 500// SearchHits specifies the list of search hits. 501type SearchHits struct { 502 TotalHits int64 `json:"total"` // total number of hits found 503 MaxScore *float64 `json:"max_score,omitempty"` // maximum score of all hits 504 Hits []*SearchHit `json:"hits,omitempty"` // the actual hits returned 505} 506 507// NestedHit is a nested innerhit 508type NestedHit struct { 509 Field string `json:"field"` 510 Offset int `json:"offset,omitempty"` 511 Child *NestedHit `json:"_nested,omitempty"` 512} 513 514// SearchHit is a single hit. 515type SearchHit struct { 516 Score *float64 `json:"_score,omitempty"` // computed score 517 Index string `json:"_index,omitempty"` // index name 518 Type string `json:"_type,omitempty"` // type meta field 519 Id string `json:"_id,omitempty"` // external or internal 520 Uid string `json:"_uid,omitempty"` // uid meta field (see MapperService.java for all meta fields) 521 Routing string `json:"_routing,omitempty"` // routing meta field 522 Parent string `json:"_parent,omitempty"` // parent meta field 523 Version *int64 `json:"_version,omitempty"` // version number, when Version is set to true in SearchService 524 Sort []interface{} `json:"sort,omitempty"` // sort information 525 Highlight SearchHitHighlight `json:"highlight,omitempty"` // highlighter information 526 Source *json.RawMessage `json:"_source,omitempty"` // stored document source 527 Fields map[string]interface{} `json:"fields,omitempty"` // returned (stored) fields 528 Explanation *SearchExplanation `json:"_explanation,omitempty"` // explains how the score was computed 529 MatchedQueries []string `json:"matched_queries,omitempty"` // matched queries 530 InnerHits map[string]*SearchHitInnerHits `json:"inner_hits,omitempty"` // inner hits with ES >= 1.5.0 531 Nested *NestedHit `json:"_nested,omitempty"` // for nested inner hits 532 533 // Shard 534 // HighlightFields 535 // SortValues 536 // MatchedFilters 537} 538 539// SearchHitInnerHits is used for inner hits. 540type SearchHitInnerHits struct { 541 Hits *SearchHits `json:"hits,omitempty"` 542} 543 544// SearchExplanation explains how the score for a hit was computed. 545// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-request-explain.html. 546type SearchExplanation struct { 547 Value float64 `json:"value"` // e.g. 1.0 548 Description string `json:"description"` // e.g. "boost" or "ConstantScore(*:*), product of:" 549 Details []SearchExplanation `json:"details,omitempty"` // recursive details 550} 551 552// Suggest 553 554// SearchSuggest is a map of suggestions. 555// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-suggesters.html. 556type SearchSuggest map[string][]SearchSuggestion 557 558// SearchSuggestion is a single search suggestion. 559// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-suggesters.html. 560type SearchSuggestion struct { 561 Text string `json:"text"` 562 Offset int `json:"offset"` 563 Length int `json:"length"` 564 Options []SearchSuggestionOption `json:"options"` 565} 566 567// SearchSuggestionOption is an option of a SearchSuggestion. 568// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-suggesters.html. 569type SearchSuggestionOption struct { 570 Text string `json:"text"` 571 Index string `json:"_index"` 572 Type string `json:"_type"` 573 Id string `json:"_id"` 574 Score float64 `json:"score"` // term and phrase suggesters uses "score" as of 6.2.4 575 ScoreUnderscore float64 `json:"_score"` // completion and context suggesters uses "_score" as of 6.2.4 576 Highlighted string `json:"highlighted"` 577 CollateMatch bool `json:"collate_match"` 578 Freq int `json:"freq"` // from TermSuggestion.Option in Java API 579 Source *json.RawMessage `json:"_source"` 580} 581 582// SearchProfile is a list of shard profiling data collected during 583// query execution in the "profile" section of a SearchResult 584type SearchProfile struct { 585 Shards []SearchProfileShardResult `json:"shards"` 586} 587 588// SearchProfileShardResult returns the profiling data for a single shard 589// accessed during the search query or aggregation. 590type SearchProfileShardResult struct { 591 ID string `json:"id"` 592 Searches []QueryProfileShardResult `json:"searches"` 593 Aggregations []ProfileResult `json:"aggregations"` 594} 595 596// QueryProfileShardResult is a container class to hold the profile results 597// for a single shard in the request. It comtains a list of query profiles, 598// a collector tree and a total rewrite tree. 599type QueryProfileShardResult struct { 600 Query []ProfileResult `json:"query,omitempty"` 601 RewriteTime int64 `json:"rewrite_time,omitempty"` 602 Collector []interface{} `json:"collector,omitempty"` 603} 604 605// CollectorResult holds the profile timings of the collectors used in the 606// search. Children's CollectorResults may be embedded inside of a parent 607// CollectorResult. 608type CollectorResult struct { 609 Name string `json:"name,omitempty"` 610 Reason string `json:"reason,omitempty"` 611 Time string `json:"time,omitempty"` 612 TimeNanos int64 `json:"time_in_nanos,omitempty"` 613 Children []CollectorResult `json:"children,omitempty"` 614} 615 616// ProfileResult is the internal representation of a profiled query, 617// corresponding to a single node in the query tree. 618type ProfileResult struct { 619 Type string `json:"type"` 620 Description string `json:"description,omitempty"` 621 NodeTime string `json:"time,omitempty"` 622 NodeTimeNanos int64 `json:"time_in_nanos,omitempty"` 623 Breakdown map[string]int64 `json:"breakdown,omitempty"` 624 Children []ProfileResult `json:"children,omitempty"` 625} 626 627// Aggregations (see search_aggs.go) 628 629// Highlighting 630 631// SearchHitHighlight is the highlight information of a search hit. 632// See https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-request-highlighting.html 633// for a general discussion of highlighting. 634type SearchHitHighlight map[string][]string 635