1// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2// See LICENSE.txt for license information. 3 4package bleveengine 5 6import ( 7 "net/http" 8 "strings" 9 10 "github.com/blevesearch/bleve" 11 "github.com/blevesearch/bleve/search/query" 12 13 "github.com/mattermost/mattermost-server/v6/model" 14 "github.com/mattermost/mattermost-server/v6/shared/mlog" 15) 16 17const DeletePostsBatchSize = 500 18const DeleteFilesBatchSize = 500 19 20func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError { 21 b.Mutex.RLock() 22 defer b.Mutex.RUnlock() 23 24 blvPost := BLVPostFromPost(post, teamId) 25 if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil { 26 return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, err.Error(), http.StatusInternalServerError) 27 } 28 return nil 29} 30 31func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { 32 channelQueries := []query.Query{} 33 for _, channel := range channels { 34 channelIdQ := bleve.NewTermQuery(channel.Id) 35 channelIdQ.SetField("ChannelId") 36 channelQueries = append(channelQueries, channelIdQ) 37 } 38 channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...) 39 40 var termQueries []query.Query 41 var notTermQueries []query.Query 42 var filters []query.Query 43 var notFilters []query.Query 44 45 typeQ := bleve.NewTermQuery("") 46 typeQ.SetField("Type") 47 filters = append(filters, typeQ) 48 49 for i, params := range searchParams { 50 var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd 51 if searchParams[0].OrTerms { 52 termOperator = query.MatchQueryOperatorOr 53 } 54 55 // Date, channels and FromUsers filters come in all 56 // searchParams iteration, and as they are global to the 57 // query, we only need to process them once 58 if i == 0 { 59 if len(params.InChannels) > 0 { 60 inChannels := []query.Query{} 61 for _, channelId := range params.InChannels { 62 channelQ := bleve.NewTermQuery(channelId) 63 channelQ.SetField("ChannelId") 64 inChannels = append(inChannels, channelQ) 65 } 66 filters = append(filters, bleve.NewDisjunctionQuery(inChannels...)) 67 } 68 69 if len(params.ExcludedChannels) > 0 { 70 excludedChannels := []query.Query{} 71 for _, channelId := range params.ExcludedChannels { 72 channelQ := bleve.NewTermQuery(channelId) 73 channelQ.SetField("ChannelId") 74 excludedChannels = append(excludedChannels, channelQ) 75 } 76 notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...)) 77 } 78 79 if len(params.FromUsers) > 0 { 80 fromUsers := []query.Query{} 81 for _, userId := range params.FromUsers { 82 userQ := bleve.NewTermQuery(userId) 83 userQ.SetField("UserId") 84 fromUsers = append(fromUsers, userQ) 85 } 86 filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...)) 87 } 88 89 if len(params.ExcludedUsers) > 0 { 90 excludedUsers := []query.Query{} 91 for _, userId := range params.ExcludedUsers { 92 userQ := bleve.NewTermQuery(userId) 93 userQ.SetField("UserId") 94 excludedUsers = append(excludedUsers, userQ) 95 } 96 notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...)) 97 } 98 99 if params.OnDate != "" { 100 before, after := params.GetOnDateMillis() 101 beforeFloat64 := float64(before) 102 afterFloat64 := float64(after) 103 onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64) 104 onDateQ.SetField("CreateAt") 105 filters = append(filters, onDateQ) 106 } else { 107 if params.AfterDate != "" || params.BeforeDate != "" { 108 var min, max *float64 109 if params.AfterDate != "" { 110 minf := float64(params.GetAfterDateMillis()) 111 min = &minf 112 } 113 114 if params.BeforeDate != "" { 115 maxf := float64(params.GetBeforeDateMillis()) 116 max = &maxf 117 } 118 119 dateQ := bleve.NewNumericRangeQuery(min, max) 120 dateQ.SetField("CreateAt") 121 filters = append(filters, dateQ) 122 } 123 124 if params.ExcludedAfterDate != "" { 125 minf := float64(params.GetExcludedAfterDateMillis()) 126 dateQ := bleve.NewNumericRangeQuery(&minf, nil) 127 dateQ.SetField("CreateAt") 128 notFilters = append(notFilters, dateQ) 129 } 130 131 if params.ExcludedBeforeDate != "" { 132 maxf := float64(params.GetExcludedBeforeDateMillis()) 133 dateQ := bleve.NewNumericRangeQuery(nil, &maxf) 134 dateQ.SetField("CreateAt") 135 notFilters = append(notFilters, dateQ) 136 } 137 138 if params.ExcludedDate != "" { 139 before, after := params.GetExcludedDateMillis() 140 beforef := float64(before) 141 afterf := float64(after) 142 onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf) 143 onDateQ.SetField("CreateAt") 144 notFilters = append(notFilters, onDateQ) 145 } 146 } 147 } 148 149 if params.IsHashtag { 150 if params.Terms != "" { 151 hashtagQ := bleve.NewMatchQuery(params.Terms) 152 hashtagQ.SetField("Hashtags") 153 hashtagQ.SetOperator(termOperator) 154 termQueries = append(termQueries, hashtagQ) 155 } else if params.ExcludedTerms != "" { 156 hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms) 157 hashtagQ.SetField("Hashtags") 158 hashtagQ.SetOperator(termOperator) 159 notTermQueries = append(notTermQueries, hashtagQ) 160 } 161 } else { 162 if params.Terms != "" { 163 terms := []string{} 164 for _, term := range strings.Split(params.Terms, " ") { 165 if strings.HasSuffix(term, "*") { 166 messageQ := bleve.NewWildcardQuery(term) 167 messageQ.SetField("Message") 168 termQueries = append(termQueries, messageQ) 169 } else { 170 terms = append(terms, term) 171 } 172 } 173 174 if len(terms) > 0 { 175 messageQ := bleve.NewMatchQuery(strings.Join(terms, " ")) 176 messageQ.SetField("Message") 177 messageQ.SetOperator(termOperator) 178 termQueries = append(termQueries, messageQ) 179 } 180 } 181 182 if params.ExcludedTerms != "" { 183 messageQ := bleve.NewMatchQuery(params.ExcludedTerms) 184 messageQ.SetField("Message") 185 messageQ.SetOperator(termOperator) 186 notTermQueries = append(notTermQueries, messageQ) 187 } 188 } 189 } 190 191 allTermsQ := bleve.NewBooleanQuery() 192 allTermsQ.AddMustNot(notTermQueries...) 193 if searchParams[0].OrTerms { 194 allTermsQ.AddShould(termQueries...) 195 } else { 196 allTermsQ.AddMust(termQueries...) 197 } 198 199 query := bleve.NewBooleanQuery() 200 query.AddMust(channelDisjunctionQ) 201 202 if len(termQueries) > 0 || len(notTermQueries) > 0 { 203 query.AddMust(allTermsQ) 204 } 205 206 if len(filters) > 0 { 207 query.AddMust(bleve.NewConjunctionQuery(filters...)) 208 } 209 if len(notFilters) > 0 { 210 query.AddMustNot(notFilters...) 211 } 212 213 search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false) 214 search.SortBy([]string{"-CreateAt"}) 215 results, err := b.PostIndex.Search(search) 216 if err != nil { 217 return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, err.Error(), http.StatusInternalServerError) 218 } 219 220 postIds := []string{} 221 matches := model.PostSearchMatches{} 222 223 for _, r := range results.Hits { 224 postIds = append(postIds, r.ID) 225 } 226 227 return postIds, matches, nil 228} 229 230func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) { 231 resultsCount := int64(0) 232 233 for { 234 // As we are deleting the posts after fetching them, we need to keep 235 // From fixed always to 0 236 searchRequest.From = 0 237 searchRequest.Size = batchSize 238 results, err := b.PostIndex.Search(searchRequest) 239 if err != nil { 240 return -1, err 241 } 242 batch := b.PostIndex.NewBatch() 243 for _, post := range results.Hits { 244 batch.Delete(post.ID) 245 } 246 if err := b.PostIndex.Batch(batch); err != nil { 247 return -1, err 248 } 249 resultsCount += int64(results.Hits.Len()) 250 if results.Hits.Len() < batchSize { 251 break 252 } 253 } 254 255 return resultsCount, nil 256} 257 258func (b *BleveEngine) DeleteChannelPosts(channelID string) *model.AppError { 259 b.Mutex.RLock() 260 defer b.Mutex.RUnlock() 261 262 query := bleve.NewTermQuery(channelID) 263 query.SetField("ChannelId") 264 search := bleve.NewSearchRequest(query) 265 deleted, err := b.deletePosts(search, DeletePostsBatchSize) 266 if err != nil { 267 return model.NewAppError("Bleveengine.DeleteChannelPosts", 268 "bleveengine.delete_channel_posts.error", nil, 269 err.Error(), http.StatusInternalServerError) 270 } 271 272 mlog.Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int64("deleted", deleted)) 273 274 return nil 275} 276 277func (b *BleveEngine) DeleteUserPosts(userID string) *model.AppError { 278 b.Mutex.RLock() 279 defer b.Mutex.RUnlock() 280 281 query := bleve.NewTermQuery(userID) 282 query.SetField("UserId") 283 search := bleve.NewSearchRequest(query) 284 deleted, err := b.deletePosts(search, DeletePostsBatchSize) 285 if err != nil { 286 return model.NewAppError("Bleveengine.DeleteUserPosts", 287 "bleveengine.delete_user_posts.error", nil, 288 err.Error(), http.StatusInternalServerError) 289 } 290 291 mlog.Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted)) 292 293 return nil 294} 295 296func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError { 297 b.Mutex.RLock() 298 defer b.Mutex.RUnlock() 299 300 if err := b.PostIndex.Delete(post.Id); err != nil { 301 return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, err.Error(), http.StatusInternalServerError) 302 } 303 return nil 304} 305 306func (b *BleveEngine) IndexChannel(channel *model.Channel) *model.AppError { 307 b.Mutex.RLock() 308 defer b.Mutex.RUnlock() 309 310 blvChannel := BLVChannelFromChannel(channel) 311 if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil { 312 return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, err.Error(), http.StatusInternalServerError) 313 } 314 return nil 315} 316 317func (b *BleveEngine) SearchChannels(teamId, term string) ([]string, *model.AppError) { 318 teamIdQ := bleve.NewTermQuery(teamId) 319 teamIdQ.SetField("TeamId") 320 queries := []query.Query{teamIdQ} 321 322 if term != "" { 323 nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term)) 324 nameSuggestQ.SetField("NameSuggest") 325 queries = append(queries, nameSuggestQ) 326 } 327 328 query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...)) 329 query.Size = model.ChannelSearchDefaultLimit 330 results, err := b.ChannelIndex.Search(query) 331 if err != nil { 332 return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, err.Error(), http.StatusInternalServerError) 333 } 334 335 channelIds := []string{} 336 for _, result := range results.Hits { 337 channelIds = append(channelIds, result.ID) 338 } 339 340 return channelIds, nil 341} 342 343func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError { 344 b.Mutex.RLock() 345 defer b.Mutex.RUnlock() 346 347 if err := b.ChannelIndex.Delete(channel.Id); err != nil { 348 return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, err.Error(), http.StatusInternalServerError) 349 } 350 return nil 351} 352 353func (b *BleveEngine) IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError { 354 b.Mutex.RLock() 355 defer b.Mutex.RUnlock() 356 357 blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds) 358 if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil { 359 return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, err.Error(), http.StatusInternalServerError) 360 } 361 return nil 362} 363 364func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { 365 if restrictedToChannels != nil && len(restrictedToChannels) == 0 { 366 return []string{}, []string{}, nil 367 } 368 369 // users in channel 370 var queries []query.Query 371 if term != "" { 372 termQ := bleve.NewPrefixQuery(strings.ToLower(term)) 373 if options.AllowFullNames { 374 termQ.SetField("SuggestionsWithFullname") 375 } else { 376 termQ.SetField("SuggestionsWithoutFullname") 377 } 378 queries = append(queries, termQ) 379 } 380 381 channelIdQ := bleve.NewTermQuery(channelId) 382 channelIdQ.SetField("ChannelsIds") 383 queries = append(queries, channelIdQ) 384 385 query := bleve.NewConjunctionQuery(queries...) 386 387 uchanSearch := bleve.NewSearchRequest(query) 388 uchanSearch.Size = options.Limit 389 uchan, err := b.UserIndex.Search(uchanSearch) 390 if err != nil { 391 return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, err.Error(), http.StatusInternalServerError) 392 } 393 394 // users not in channel 395 boolQ := bleve.NewBooleanQuery() 396 397 if term != "" { 398 termQ := bleve.NewPrefixQuery(strings.ToLower(term)) 399 if options.AllowFullNames { 400 termQ.SetField("SuggestionsWithFullname") 401 } else { 402 termQ.SetField("SuggestionsWithoutFullname") 403 } 404 boolQ.AddMust(termQ) 405 } 406 407 teamIdQ := bleve.NewTermQuery(teamId) 408 teamIdQ.SetField("TeamsIds") 409 boolQ.AddMust(teamIdQ) 410 411 outsideChannelIdQ := bleve.NewTermQuery(channelId) 412 outsideChannelIdQ.SetField("ChannelsIds") 413 boolQ.AddMustNot(outsideChannelIdQ) 414 415 if len(restrictedToChannels) > 0 { 416 restrictedChannelsQ := bleve.NewDisjunctionQuery() 417 for _, channelId := range restrictedToChannels { 418 restrictedChannelQ := bleve.NewTermQuery(channelId) 419 restrictedChannelsQ.AddQuery(restrictedChannelQ) 420 } 421 boolQ.AddMust(restrictedChannelsQ) 422 } 423 424 nuchanSearch := bleve.NewSearchRequest(boolQ) 425 nuchanSearch.Size = options.Limit 426 nuchan, err := b.UserIndex.Search(nuchanSearch) 427 if err != nil { 428 return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, err.Error(), http.StatusInternalServerError) 429 } 430 431 uchanIds := []string{} 432 for _, result := range uchan.Hits { 433 uchanIds = append(uchanIds, result.ID) 434 } 435 436 nuchanIds := []string{} 437 for _, result := range nuchan.Hits { 438 nuchanIds = append(nuchanIds, result.ID) 439 } 440 441 return uchanIds, nuchanIds, nil 442} 443 444func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { 445 if restrictedToChannels != nil && len(restrictedToChannels) == 0 { 446 return []string{}, nil 447 } 448 449 var rootQ query.Query 450 if term == "" && teamId == "" && restrictedToChannels == nil { 451 rootQ = bleve.NewMatchAllQuery() 452 } else { 453 boolQ := bleve.NewBooleanQuery() 454 455 if term != "" { 456 termQ := bleve.NewPrefixQuery(strings.ToLower(term)) 457 if options.AllowFullNames { 458 termQ.SetField("SuggestionsWithFullname") 459 } else { 460 termQ.SetField("SuggestionsWithoutFullname") 461 } 462 boolQ.AddMust(termQ) 463 } 464 465 if len(restrictedToChannels) > 0 { 466 // restricted channels are already filtered by team, so we 467 // can search only those matches 468 restrictedChannelsQ := []query.Query{} 469 for _, channelId := range restrictedToChannels { 470 channelIdQ := bleve.NewTermQuery(channelId) 471 channelIdQ.SetField("ChannelsIds") 472 restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ) 473 } 474 boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...)) 475 } else { 476 // this means that we only need to restrict by team 477 if teamId != "" { 478 teamIdQ := bleve.NewTermQuery(teamId) 479 teamIdQ.SetField("TeamsIds") 480 boolQ.AddMust(teamIdQ) 481 } 482 } 483 484 rootQ = boolQ 485 } 486 487 search := bleve.NewSearchRequest(rootQ) 488 search.Size = options.Limit 489 results, err := b.UserIndex.Search(search) 490 if err != nil { 491 return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, err.Error(), http.StatusInternalServerError) 492 } 493 494 usersIds := []string{} 495 for _, r := range results.Hits { 496 usersIds = append(usersIds, r.ID) 497 } 498 499 return usersIds, nil 500} 501 502func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError { 503 b.Mutex.RLock() 504 defer b.Mutex.RUnlock() 505 506 if err := b.UserIndex.Delete(user.Id); err != nil { 507 return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, err.Error(), http.StatusInternalServerError) 508 } 509 return nil 510} 511 512func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError { 513 b.Mutex.RLock() 514 defer b.Mutex.RUnlock() 515 516 blvFile := BLVFileFromFileInfo(file, channelId) 517 if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil { 518 return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, err.Error(), http.StatusInternalServerError) 519 } 520 return nil 521} 522 523func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) { 524 channelQueries := []query.Query{} 525 for _, channel := range channels { 526 channelIdQ := bleve.NewTermQuery(channel.Id) 527 channelIdQ.SetField("ChannelId") 528 channelQueries = append(channelQueries, channelIdQ) 529 } 530 channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...) 531 532 var termQueries []query.Query 533 var notTermQueries []query.Query 534 var filters []query.Query 535 var notFilters []query.Query 536 537 for i, params := range searchParams { 538 var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd 539 if searchParams[0].OrTerms { 540 termOperator = query.MatchQueryOperatorOr 541 } 542 543 // Date, channels and FromUsers filters come in all 544 // searchParams iteration, and as they are global to the 545 // query, we only need to process them once 546 if i == 0 { 547 if len(params.InChannels) > 0 { 548 inChannels := []query.Query{} 549 for _, channelId := range params.InChannels { 550 channelQ := bleve.NewTermQuery(channelId) 551 channelQ.SetField("ChannelId") 552 inChannels = append(inChannels, channelQ) 553 } 554 filters = append(filters, bleve.NewDisjunctionQuery(inChannels...)) 555 } 556 557 if len(params.ExcludedChannels) > 0 { 558 excludedChannels := []query.Query{} 559 for _, channelId := range params.ExcludedChannels { 560 channelQ := bleve.NewTermQuery(channelId) 561 channelQ.SetField("ChannelId") 562 excludedChannels = append(excludedChannels, channelQ) 563 } 564 notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...)) 565 } 566 567 if len(params.FromUsers) > 0 { 568 fromUsers := []query.Query{} 569 for _, userId := range params.FromUsers { 570 userQ := bleve.NewTermQuery(userId) 571 userQ.SetField("CreatorId") 572 fromUsers = append(fromUsers, userQ) 573 } 574 filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...)) 575 } 576 577 if len(params.ExcludedUsers) > 0 { 578 excludedUsers := []query.Query{} 579 for _, userId := range params.ExcludedUsers { 580 userQ := bleve.NewTermQuery(userId) 581 userQ.SetField("CreatorId") 582 excludedUsers = append(excludedUsers, userQ) 583 } 584 notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...)) 585 } 586 587 if len(params.Extensions) > 0 { 588 extensions := []query.Query{} 589 for _, extension := range params.Extensions { 590 extensionQ := bleve.NewTermQuery(extension) 591 extensionQ.SetField("Extension") 592 extensions = append(extensions, extensionQ) 593 } 594 filters = append(filters, bleve.NewDisjunctionQuery(extensions...)) 595 } 596 597 if len(params.ExcludedExtensions) > 0 { 598 excludedExtensions := []query.Query{} 599 for _, extension := range params.ExcludedExtensions { 600 extensionQ := bleve.NewTermQuery(extension) 601 extensionQ.SetField("Extension") 602 excludedExtensions = append(excludedExtensions, extensionQ) 603 } 604 notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...)) 605 } 606 607 if params.OnDate != "" { 608 before, after := params.GetOnDateMillis() 609 beforeFloat64 := float64(before) 610 afterFloat64 := float64(after) 611 onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64) 612 onDateQ.SetField("CreateAt") 613 filters = append(filters, onDateQ) 614 } else { 615 if params.AfterDate != "" || params.BeforeDate != "" { 616 var min, max *float64 617 if params.AfterDate != "" { 618 minf := float64(params.GetAfterDateMillis()) 619 min = &minf 620 } 621 622 if params.BeforeDate != "" { 623 maxf := float64(params.GetBeforeDateMillis()) 624 max = &maxf 625 } 626 627 dateQ := bleve.NewNumericRangeQuery(min, max) 628 dateQ.SetField("CreateAt") 629 filters = append(filters, dateQ) 630 } 631 632 if params.ExcludedAfterDate != "" { 633 minf := float64(params.GetExcludedAfterDateMillis()) 634 dateQ := bleve.NewNumericRangeQuery(&minf, nil) 635 dateQ.SetField("CreateAt") 636 notFilters = append(notFilters, dateQ) 637 } 638 639 if params.ExcludedBeforeDate != "" { 640 maxf := float64(params.GetExcludedBeforeDateMillis()) 641 dateQ := bleve.NewNumericRangeQuery(nil, &maxf) 642 dateQ.SetField("CreateAt") 643 notFilters = append(notFilters, dateQ) 644 } 645 646 if params.ExcludedDate != "" { 647 before, after := params.GetExcludedDateMillis() 648 beforef := float64(before) 649 afterf := float64(after) 650 onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf) 651 onDateQ.SetField("CreateAt") 652 notFilters = append(notFilters, onDateQ) 653 } 654 } 655 } 656 657 if params.Terms != "" { 658 terms := []string{} 659 for _, term := range strings.Split(params.Terms, " ") { 660 if strings.HasSuffix(term, "*") { 661 nameQ := bleve.NewWildcardQuery(term) 662 nameQ.SetField("Name") 663 contentQ := bleve.NewWildcardQuery(term) 664 contentQ.SetField("Content") 665 termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) 666 } else { 667 terms = append(terms, term) 668 } 669 } 670 671 if len(terms) > 0 { 672 nameQ := bleve.NewMatchQuery(strings.Join(terms, " ")) 673 nameQ.SetField("Name") 674 nameQ.SetOperator(termOperator) 675 contentQ := bleve.NewMatchQuery(strings.Join(terms, " ")) 676 contentQ.SetField("Content") 677 contentQ.SetOperator(termOperator) 678 termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) 679 } 680 } 681 682 if params.ExcludedTerms != "" { 683 nameQ := bleve.NewMatchQuery(params.ExcludedTerms) 684 nameQ.SetField("Name") 685 nameQ.SetOperator(termOperator) 686 contentQ := bleve.NewMatchQuery(params.ExcludedTerms) 687 contentQ.SetField("Content") 688 contentQ.SetOperator(termOperator) 689 notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) 690 } 691 } 692 693 allTermsQ := bleve.NewBooleanQuery() 694 allTermsQ.AddMustNot(notTermQueries...) 695 if searchParams[0].OrTerms { 696 allTermsQ.AddShould(termQueries...) 697 } else { 698 allTermsQ.AddMust(termQueries...) 699 } 700 701 query := bleve.NewBooleanQuery() 702 query.AddMust(channelDisjunctionQ) 703 704 if len(termQueries) > 0 || len(notTermQueries) > 0 { 705 query.AddMust(allTermsQ) 706 } 707 708 if len(filters) > 0 { 709 query.AddMust(bleve.NewConjunctionQuery(filters...)) 710 } 711 if len(notFilters) > 0 { 712 query.AddMustNot(notFilters...) 713 } 714 715 search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false) 716 search.SortBy([]string{"-CreateAt"}) 717 results, err := b.FileIndex.Search(search) 718 if err != nil { 719 return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, err.Error(), http.StatusInternalServerError) 720 } 721 722 fileIds := []string{} 723 724 for _, r := range results.Hits { 725 fileIds = append(fileIds, r.ID) 726 } 727 728 return fileIds, nil 729} 730 731func (b *BleveEngine) DeleteFile(fileID string) *model.AppError { 732 b.Mutex.RLock() 733 defer b.Mutex.RUnlock() 734 735 if err := b.FileIndex.Delete(fileID); err != nil { 736 return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, err.Error(), http.StatusInternalServerError) 737 } 738 return nil 739} 740 741func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) { 742 resultsCount := int64(0) 743 744 for { 745 // As we are deleting the files after fetching them, we need to keep 746 // From fixed always to 0 747 searchRequest.From = 0 748 searchRequest.Size = batchSize 749 results, err := b.FileIndex.Search(searchRequest) 750 if err != nil { 751 return -1, err 752 } 753 batch := b.FileIndex.NewBatch() 754 for _, file := range results.Hits { 755 batch.Delete(file.ID) 756 } 757 if err := b.FileIndex.Batch(batch); err != nil { 758 return -1, err 759 } 760 resultsCount += int64(results.Hits.Len()) 761 if results.Hits.Len() < batchSize { 762 break 763 } 764 } 765 766 return resultsCount, nil 767} 768 769func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError { 770 b.Mutex.RLock() 771 defer b.Mutex.RUnlock() 772 773 query := bleve.NewTermQuery(userID) 774 query.SetField("CreatorId") 775 search := bleve.NewSearchRequest(query) 776 deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) 777 if err != nil { 778 return model.NewAppError("Bleveengine.DeleteUserFiles", 779 "bleveengine.delete_user_files.error", nil, 780 err.Error(), http.StatusInternalServerError) 781 } 782 783 mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted)) 784 785 return nil 786} 787 788func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError { 789 b.Mutex.RLock() 790 defer b.Mutex.RUnlock() 791 792 query := bleve.NewTermQuery(postID) 793 query.SetField("PostId") 794 search := bleve.NewSearchRequest(query) 795 deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) 796 if err != nil { 797 return model.NewAppError("Bleveengine.DeletePostFiles", 798 "bleveengine.delete_post_files.error", nil, 799 err.Error(), http.StatusInternalServerError) 800 } 801 802 mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted)) 803 804 return nil 805} 806 807func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError { 808 b.Mutex.RLock() 809 defer b.Mutex.RUnlock() 810 811 endTimeFloat := float64(endTime) 812 query := bleve.NewNumericRangeQuery(nil, &endTimeFloat) 813 query.SetField("CreateAt") 814 search := bleve.NewSearchRequestOptions(query, int(limit), 0, false) 815 search.SortBy([]string{"-CreateAt"}) 816 817 deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) 818 if err != nil { 819 return model.NewAppError("Bleveengine.DeleteFilesBatch", 820 "bleveengine.delete_files_batch.error", nil, 821 err.Error(), http.StatusInternalServerError) 822 } 823 824 mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted)) 825 826 return nil 827} 828