1// Copyright (c) 2014 Couchbase, Inc. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bleve 16 17import ( 18 "encoding/json" 19 "fmt" 20 "time" 21 22 "github.com/blevesearch/bleve/analysis" 23 "github.com/blevesearch/bleve/analysis/datetime/optional" 24 "github.com/blevesearch/bleve/registry" 25 "github.com/blevesearch/bleve/search" 26 "github.com/blevesearch/bleve/search/query" 27) 28 29var cache = registry.NewCache() 30 31const defaultDateTimeParser = optional.Name 32 33type numericRange struct { 34 Name string `json:"name,omitempty"` 35 Min *float64 `json:"min,omitempty"` 36 Max *float64 `json:"max,omitempty"` 37} 38 39type dateTimeRange struct { 40 Name string `json:"name,omitempty"` 41 Start time.Time `json:"start,omitempty"` 42 End time.Time `json:"end,omitempty"` 43 startString *string 44 endString *string 45} 46 47func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time) { 48 start = dr.Start 49 if dr.Start.IsZero() && dr.startString != nil { 50 s, err := dateTimeParser.ParseDateTime(*dr.startString) 51 if err == nil { 52 start = s 53 } 54 } 55 end = dr.End 56 if dr.End.IsZero() && dr.endString != nil { 57 e, err := dateTimeParser.ParseDateTime(*dr.endString) 58 if err == nil { 59 end = e 60 } 61 } 62 return start, end 63} 64 65func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { 66 var temp struct { 67 Name string `json:"name,omitempty"` 68 Start *string `json:"start,omitempty"` 69 End *string `json:"end,omitempty"` 70 } 71 72 err := json.Unmarshal(input, &temp) 73 if err != nil { 74 return err 75 } 76 77 dr.Name = temp.Name 78 if temp.Start != nil { 79 dr.startString = temp.Start 80 } 81 if temp.End != nil { 82 dr.endString = temp.End 83 } 84 85 return nil 86} 87 88func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { 89 rv := map[string]interface{}{ 90 "name": dr.Name, 91 "start": dr.Start, 92 "end": dr.End, 93 } 94 if dr.Start.IsZero() && dr.startString != nil { 95 rv["start"] = dr.startString 96 } 97 if dr.End.IsZero() && dr.endString != nil { 98 rv["end"] = dr.endString 99 } 100 return json.Marshal(rv) 101} 102 103// A FacetRequest describes a facet or aggregation 104// of the result document set you would like to be 105// built. 106type FacetRequest struct { 107 Size int `json:"size"` 108 Field string `json:"field"` 109 NumericRanges []*numericRange `json:"numeric_ranges,omitempty"` 110 DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` 111} 112 113func (fr *FacetRequest) Validate() error { 114 nrCount := len(fr.NumericRanges) 115 drCount := len(fr.DateTimeRanges) 116 if nrCount > 0 && drCount > 0 { 117 return fmt.Errorf("facet can only conain numeric ranges or date ranges, not both") 118 } 119 120 if nrCount > 0 { 121 nrNames := map[string]interface{}{} 122 for _, nr := range fr.NumericRanges { 123 if _, ok := nrNames[nr.Name]; ok { 124 return fmt.Errorf("numeric ranges contains duplicate name '%s'", nr.Name) 125 } 126 nrNames[nr.Name] = struct{}{} 127 if nr.Min == nil && nr.Max == nil { 128 return fmt.Errorf("numeric range query must specify either min, max or both for range name '%s'", nr.Name) 129 } 130 } 131 132 } else { 133 dateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser) 134 if err != nil { 135 return err 136 } 137 drNames := map[string]interface{}{} 138 for _, dr := range fr.DateTimeRanges { 139 if _, ok := drNames[dr.Name]; ok { 140 return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) 141 } 142 drNames[dr.Name] = struct{}{} 143 start, end := dr.ParseDates(dateTimeParser) 144 if start.IsZero() && end.IsZero() { 145 return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) 146 } 147 } 148 } 149 return nil 150} 151 152// NewFacetRequest creates a facet on the specified 153// field that limits the number of entries to the 154// specified size. 155func NewFacetRequest(field string, size int) *FacetRequest { 156 return &FacetRequest{ 157 Field: field, 158 Size: size, 159 } 160} 161 162// AddDateTimeRange adds a bucket to a field 163// containing date values. Documents with a 164// date value falling into this range are tabulated 165// as part of this bucket/range. 166func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { 167 if fr.DateTimeRanges == nil { 168 fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) 169 } 170 fr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end}) 171} 172 173// AddDateTimeRangeString adds a bucket to a field 174// containing date values. 175func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { 176 if fr.DateTimeRanges == nil { 177 fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) 178 } 179 fr.DateTimeRanges = append(fr.DateTimeRanges, 180 &dateTimeRange{Name: name, startString: start, endString: end}) 181} 182 183// AddNumericRange adds a bucket to a field 184// containing numeric values. Documents with a 185// numeric value falling into this range are 186// tabulated as part of this bucket/range. 187func (fr *FacetRequest) AddNumericRange(name string, min, max *float64) { 188 if fr.NumericRanges == nil { 189 fr.NumericRanges = make([]*numericRange, 0, 1) 190 } 191 fr.NumericRanges = append(fr.NumericRanges, &numericRange{Name: name, Min: min, Max: max}) 192} 193 194// FacetsRequest groups together all the 195// FacetRequest objects for a single query. 196type FacetsRequest map[string]*FacetRequest 197 198func (fr FacetsRequest) Validate() error { 199 for _, v := range fr { 200 err := v.Validate() 201 if err != nil { 202 return err 203 } 204 } 205 return nil 206} 207 208// HighlightRequest describes how field matches 209// should be highlighted. 210type HighlightRequest struct { 211 Style *string `json:"style"` 212 Fields []string `json:"fields"` 213} 214 215// NewHighlight creates a default 216// HighlightRequest. 217func NewHighlight() *HighlightRequest { 218 return &HighlightRequest{} 219} 220 221// NewHighlightWithStyle creates a HighlightRequest 222// with an alternate style. 223func NewHighlightWithStyle(style string) *HighlightRequest { 224 return &HighlightRequest{ 225 Style: &style, 226 } 227} 228 229func (h *HighlightRequest) AddField(field string) { 230 if h.Fields == nil { 231 h.Fields = make([]string, 0, 1) 232 } 233 h.Fields = append(h.Fields, field) 234} 235 236// A SearchRequest describes all the parameters 237// needed to search the index. 238// Query is required. 239// Size/From describe how much and which part of the 240// result set to return. 241// Highlight describes optional search result 242// highlighting. 243// Fields describes a list of field values which 244// should be retrieved for result documents, provided they 245// were stored while indexing. 246// Facets describe the set of facets to be computed. 247// Explain triggers inclusion of additional search 248// result score explanations. 249// Sort describes the desired order for the results to be returned. 250// 251// A special field named "*" can be used to return all fields. 252type SearchRequest struct { 253 Query query.Query `json:"query"` 254 Size int `json:"size"` 255 From int `json:"from"` 256 Highlight *HighlightRequest `json:"highlight"` 257 Fields []string `json:"fields"` 258 Facets FacetsRequest `json:"facets"` 259 Explain bool `json:"explain"` 260 Sort search.SortOrder `json:"sort"` 261 IncludeLocations bool `json:"includeLocations"` 262} 263 264func (r *SearchRequest) Validate() error { 265 if srq, ok := r.Query.(query.ValidatableQuery); ok { 266 err := srq.Validate() 267 if err != nil { 268 return err 269 } 270 } 271 272 return r.Facets.Validate() 273} 274 275// AddFacet adds a FacetRequest to this SearchRequest 276func (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) { 277 if r.Facets == nil { 278 r.Facets = make(FacetsRequest, 1) 279 } 280 r.Facets[facetName] = f 281} 282 283// SortBy changes the request to use the requested sort order 284// this form uses the simplified syntax with an array of strings 285// each string can either be a field name 286// or the magic value _id and _score which refer to the doc id and search score 287// any of these values can optionally be prefixed with - to reverse the order 288func (r *SearchRequest) SortBy(order []string) { 289 so := search.ParseSortOrderStrings(order) 290 r.Sort = so 291} 292 293// SortByCustom changes the request to use the requested sort order 294func (r *SearchRequest) SortByCustom(order search.SortOrder) { 295 r.Sort = order 296} 297 298// UnmarshalJSON deserializes a JSON representation of 299// a SearchRequest 300func (r *SearchRequest) UnmarshalJSON(input []byte) error { 301 var temp struct { 302 Q json.RawMessage `json:"query"` 303 Size *int `json:"size"` 304 From int `json:"from"` 305 Highlight *HighlightRequest `json:"highlight"` 306 Fields []string `json:"fields"` 307 Facets FacetsRequest `json:"facets"` 308 Explain bool `json:"explain"` 309 Sort []json.RawMessage `json:"sort"` 310 IncludeLocations bool `json:"includeLocations"` 311 } 312 313 err := json.Unmarshal(input, &temp) 314 if err != nil { 315 return err 316 } 317 318 if temp.Size == nil { 319 r.Size = 10 320 } else { 321 r.Size = *temp.Size 322 } 323 if temp.Sort == nil { 324 r.Sort = search.SortOrder{&search.SortScore{Desc: true}} 325 } else { 326 r.Sort, err = search.ParseSortOrderJSON(temp.Sort) 327 if err != nil { 328 return err 329 } 330 } 331 r.From = temp.From 332 r.Explain = temp.Explain 333 r.Highlight = temp.Highlight 334 r.Fields = temp.Fields 335 r.Facets = temp.Facets 336 r.IncludeLocations = temp.IncludeLocations 337 r.Query, err = query.ParseQuery(temp.Q) 338 if err != nil { 339 return err 340 } 341 342 if r.Size < 0 { 343 r.Size = 10 344 } 345 if r.From < 0 { 346 r.From = 0 347 } 348 349 return nil 350 351} 352 353// NewSearchRequest creates a new SearchRequest 354// for the Query, using default values for all 355// other search parameters. 356func NewSearchRequest(q query.Query) *SearchRequest { 357 return NewSearchRequestOptions(q, 10, 0, false) 358} 359 360// NewSearchRequestOptions creates a new SearchRequest 361// for the Query, with the requested size, from 362// and explanation search parameters. 363// By default results are ordered by score, descending. 364func NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest { 365 return &SearchRequest{ 366 Query: q, 367 Size: size, 368 From: from, 369 Explain: explain, 370 Sort: search.SortOrder{&search.SortScore{Desc: true}}, 371 } 372} 373 374// IndexErrMap tracks errors with the name of the index where it occurred 375type IndexErrMap map[string]error 376 377// MarshalJSON seralizes the error into a string for JSON consumption 378func (iem IndexErrMap) MarshalJSON() ([]byte, error) { 379 tmp := make(map[string]string, len(iem)) 380 for k, v := range iem { 381 tmp[k] = v.Error() 382 } 383 return json.Marshal(tmp) 384} 385 386func (iem IndexErrMap) UnmarshalJSON(data []byte) error { 387 var tmp map[string]string 388 err := json.Unmarshal(data, &tmp) 389 if err != nil { 390 return err 391 } 392 for k, v := range tmp { 393 iem[k] = fmt.Errorf("%s", v) 394 } 395 return nil 396} 397 398// SearchStatus is a secion in the SearchResult reporting how many 399// underlying indexes were queried, how many were successful/failed 400// and a map of any errors that were encountered 401type SearchStatus struct { 402 Total int `json:"total"` 403 Failed int `json:"failed"` 404 Successful int `json:"successful"` 405 Errors IndexErrMap `json:"errors,omitempty"` 406} 407 408// Merge will merge together multiple SearchStatuses during a MultiSearch 409func (ss *SearchStatus) Merge(other *SearchStatus) { 410 ss.Total += other.Total 411 ss.Failed += other.Failed 412 ss.Successful += other.Successful 413 if len(other.Errors) > 0 { 414 if ss.Errors == nil { 415 ss.Errors = make(map[string]error) 416 } 417 for otherIndex, otherError := range other.Errors { 418 ss.Errors[otherIndex] = otherError 419 } 420 } 421} 422 423// A SearchResult describes the results of executing 424// a SearchRequest. 425type SearchResult struct { 426 Status *SearchStatus `json:"status"` 427 Request *SearchRequest `json:"request"` 428 Hits search.DocumentMatchCollection `json:"hits"` 429 Total uint64 `json:"total_hits"` 430 MaxScore float64 `json:"max_score"` 431 Took time.Duration `json:"took"` 432 Facets search.FacetResults `json:"facets"` 433} 434 435func (sr *SearchResult) String() string { 436 rv := "" 437 if sr.Total > 0 { 438 if sr.Request.Size > 0 { 439 rv = fmt.Sprintf("%d matches, showing %d through %d, took %s\n", sr.Total, sr.Request.From+1, sr.Request.From+len(sr.Hits), sr.Took) 440 for i, hit := range sr.Hits { 441 rv += fmt.Sprintf("%5d. %s (%f)\n", i+sr.Request.From+1, hit.ID, hit.Score) 442 for fragmentField, fragments := range hit.Fragments { 443 rv += fmt.Sprintf("\t%s\n", fragmentField) 444 for _, fragment := range fragments { 445 rv += fmt.Sprintf("\t\t%s\n", fragment) 446 } 447 } 448 for otherFieldName, otherFieldValue := range hit.Fields { 449 if _, ok := hit.Fragments[otherFieldName]; !ok { 450 rv += fmt.Sprintf("\t%s\n", otherFieldName) 451 rv += fmt.Sprintf("\t\t%v\n", otherFieldValue) 452 } 453 } 454 } 455 } else { 456 rv = fmt.Sprintf("%d matches, took %s\n", sr.Total, sr.Took) 457 } 458 } else { 459 rv = "No matches" 460 } 461 if len(sr.Facets) > 0 { 462 rv += fmt.Sprintf("Facets:\n") 463 for fn, f := range sr.Facets { 464 rv += fmt.Sprintf("%s(%d)\n", fn, f.Total) 465 for _, t := range f.Terms { 466 rv += fmt.Sprintf("\t%s(%d)\n", t.Term, t.Count) 467 } 468 if f.Other != 0 { 469 rv += fmt.Sprintf("\tOther(%d)\n", f.Other) 470 } 471 } 472 } 473 return rv 474} 475 476// Merge will merge together multiple SearchResults during a MultiSearch 477func (sr *SearchResult) Merge(other *SearchResult) { 478 sr.Status.Merge(other.Status) 479 sr.Hits = append(sr.Hits, other.Hits...) 480 sr.Total += other.Total 481 if other.MaxScore > sr.MaxScore { 482 sr.MaxScore = other.MaxScore 483 } 484 if sr.Facets == nil && len(other.Facets) != 0 { 485 sr.Facets = other.Facets 486 return 487 } 488 489 sr.Facets.Merge(other.Facets) 490} 491