1// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2// See LICENSE.txt for license information.
3
4package bleveengine
5
6import (
7	"net/http"
8	"strings"
9
10	"github.com/blevesearch/bleve"
11	"github.com/blevesearch/bleve/search/query"
12
13	"github.com/mattermost/mattermost-server/v6/model"
14	"github.com/mattermost/mattermost-server/v6/shared/mlog"
15)
16
17const DeletePostsBatchSize = 500
18const DeleteFilesBatchSize = 500
19
20func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
21	b.Mutex.RLock()
22	defer b.Mutex.RUnlock()
23
24	blvPost := BLVPostFromPost(post, teamId)
25	if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil {
26		return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, err.Error(), http.StatusInternalServerError)
27	}
28	return nil
29}
30
31func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
32	channelQueries := []query.Query{}
33	for _, channel := range channels {
34		channelIdQ := bleve.NewTermQuery(channel.Id)
35		channelIdQ.SetField("ChannelId")
36		channelQueries = append(channelQueries, channelIdQ)
37	}
38	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
39
40	var termQueries []query.Query
41	var notTermQueries []query.Query
42	var filters []query.Query
43	var notFilters []query.Query
44
45	typeQ := bleve.NewTermQuery("")
46	typeQ.SetField("Type")
47	filters = append(filters, typeQ)
48
49	for i, params := range searchParams {
50		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
51		if searchParams[0].OrTerms {
52			termOperator = query.MatchQueryOperatorOr
53		}
54
55		// Date, channels and FromUsers filters come in all
56		// searchParams iteration, and as they are global to the
57		// query, we only need to process them once
58		if i == 0 {
59			if len(params.InChannels) > 0 {
60				inChannels := []query.Query{}
61				for _, channelId := range params.InChannels {
62					channelQ := bleve.NewTermQuery(channelId)
63					channelQ.SetField("ChannelId")
64					inChannels = append(inChannels, channelQ)
65				}
66				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
67			}
68
69			if len(params.ExcludedChannels) > 0 {
70				excludedChannels := []query.Query{}
71				for _, channelId := range params.ExcludedChannels {
72					channelQ := bleve.NewTermQuery(channelId)
73					channelQ.SetField("ChannelId")
74					excludedChannels = append(excludedChannels, channelQ)
75				}
76				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
77			}
78
79			if len(params.FromUsers) > 0 {
80				fromUsers := []query.Query{}
81				for _, userId := range params.FromUsers {
82					userQ := bleve.NewTermQuery(userId)
83					userQ.SetField("UserId")
84					fromUsers = append(fromUsers, userQ)
85				}
86				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
87			}
88
89			if len(params.ExcludedUsers) > 0 {
90				excludedUsers := []query.Query{}
91				for _, userId := range params.ExcludedUsers {
92					userQ := bleve.NewTermQuery(userId)
93					userQ.SetField("UserId")
94					excludedUsers = append(excludedUsers, userQ)
95				}
96				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
97			}
98
99			if params.OnDate != "" {
100				before, after := params.GetOnDateMillis()
101				beforeFloat64 := float64(before)
102				afterFloat64 := float64(after)
103				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
104				onDateQ.SetField("CreateAt")
105				filters = append(filters, onDateQ)
106			} else {
107				if params.AfterDate != "" || params.BeforeDate != "" {
108					var min, max *float64
109					if params.AfterDate != "" {
110						minf := float64(params.GetAfterDateMillis())
111						min = &minf
112					}
113
114					if params.BeforeDate != "" {
115						maxf := float64(params.GetBeforeDateMillis())
116						max = &maxf
117					}
118
119					dateQ := bleve.NewNumericRangeQuery(min, max)
120					dateQ.SetField("CreateAt")
121					filters = append(filters, dateQ)
122				}
123
124				if params.ExcludedAfterDate != "" {
125					minf := float64(params.GetExcludedAfterDateMillis())
126					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
127					dateQ.SetField("CreateAt")
128					notFilters = append(notFilters, dateQ)
129				}
130
131				if params.ExcludedBeforeDate != "" {
132					maxf := float64(params.GetExcludedBeforeDateMillis())
133					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
134					dateQ.SetField("CreateAt")
135					notFilters = append(notFilters, dateQ)
136				}
137
138				if params.ExcludedDate != "" {
139					before, after := params.GetExcludedDateMillis()
140					beforef := float64(before)
141					afterf := float64(after)
142					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
143					onDateQ.SetField("CreateAt")
144					notFilters = append(notFilters, onDateQ)
145				}
146			}
147		}
148
149		if params.IsHashtag {
150			if params.Terms != "" {
151				hashtagQ := bleve.NewMatchQuery(params.Terms)
152				hashtagQ.SetField("Hashtags")
153				hashtagQ.SetOperator(termOperator)
154				termQueries = append(termQueries, hashtagQ)
155			} else if params.ExcludedTerms != "" {
156				hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms)
157				hashtagQ.SetField("Hashtags")
158				hashtagQ.SetOperator(termOperator)
159				notTermQueries = append(notTermQueries, hashtagQ)
160			}
161		} else {
162			if params.Terms != "" {
163				terms := []string{}
164				for _, term := range strings.Split(params.Terms, " ") {
165					if strings.HasSuffix(term, "*") {
166						messageQ := bleve.NewWildcardQuery(term)
167						messageQ.SetField("Message")
168						termQueries = append(termQueries, messageQ)
169					} else {
170						terms = append(terms, term)
171					}
172				}
173
174				if len(terms) > 0 {
175					messageQ := bleve.NewMatchQuery(strings.Join(terms, " "))
176					messageQ.SetField("Message")
177					messageQ.SetOperator(termOperator)
178					termQueries = append(termQueries, messageQ)
179				}
180			}
181
182			if params.ExcludedTerms != "" {
183				messageQ := bleve.NewMatchQuery(params.ExcludedTerms)
184				messageQ.SetField("Message")
185				messageQ.SetOperator(termOperator)
186				notTermQueries = append(notTermQueries, messageQ)
187			}
188		}
189	}
190
191	allTermsQ := bleve.NewBooleanQuery()
192	allTermsQ.AddMustNot(notTermQueries...)
193	if searchParams[0].OrTerms {
194		allTermsQ.AddShould(termQueries...)
195	} else {
196		allTermsQ.AddMust(termQueries...)
197	}
198
199	query := bleve.NewBooleanQuery()
200	query.AddMust(channelDisjunctionQ)
201
202	if len(termQueries) > 0 || len(notTermQueries) > 0 {
203		query.AddMust(allTermsQ)
204	}
205
206	if len(filters) > 0 {
207		query.AddMust(bleve.NewConjunctionQuery(filters...))
208	}
209	if len(notFilters) > 0 {
210		query.AddMustNot(notFilters...)
211	}
212
213	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
214	search.SortBy([]string{"-CreateAt"})
215	results, err := b.PostIndex.Search(search)
216	if err != nil {
217		return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, err.Error(), http.StatusInternalServerError)
218	}
219
220	postIds := []string{}
221	matches := model.PostSearchMatches{}
222
223	for _, r := range results.Hits {
224		postIds = append(postIds, r.ID)
225	}
226
227	return postIds, matches, nil
228}
229
230func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
231	resultsCount := int64(0)
232
233	for {
234		// As we are deleting the posts after fetching them, we need to keep
235		// From fixed always to 0
236		searchRequest.From = 0
237		searchRequest.Size = batchSize
238		results, err := b.PostIndex.Search(searchRequest)
239		if err != nil {
240			return -1, err
241		}
242		batch := b.PostIndex.NewBatch()
243		for _, post := range results.Hits {
244			batch.Delete(post.ID)
245		}
246		if err := b.PostIndex.Batch(batch); err != nil {
247			return -1, err
248		}
249		resultsCount += int64(results.Hits.Len())
250		if results.Hits.Len() < batchSize {
251			break
252		}
253	}
254
255	return resultsCount, nil
256}
257
258func (b *BleveEngine) DeleteChannelPosts(channelID string) *model.AppError {
259	b.Mutex.RLock()
260	defer b.Mutex.RUnlock()
261
262	query := bleve.NewTermQuery(channelID)
263	query.SetField("ChannelId")
264	search := bleve.NewSearchRequest(query)
265	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
266	if err != nil {
267		return model.NewAppError("Bleveengine.DeleteChannelPosts",
268			"bleveengine.delete_channel_posts.error", nil,
269			err.Error(), http.StatusInternalServerError)
270	}
271
272	mlog.Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int64("deleted", deleted))
273
274	return nil
275}
276
277func (b *BleveEngine) DeleteUserPosts(userID string) *model.AppError {
278	b.Mutex.RLock()
279	defer b.Mutex.RUnlock()
280
281	query := bleve.NewTermQuery(userID)
282	query.SetField("UserId")
283	search := bleve.NewSearchRequest(query)
284	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
285	if err != nil {
286		return model.NewAppError("Bleveengine.DeleteUserPosts",
287			"bleveengine.delete_user_posts.error", nil,
288			err.Error(), http.StatusInternalServerError)
289	}
290
291	mlog.Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
292
293	return nil
294}
295
296func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError {
297	b.Mutex.RLock()
298	defer b.Mutex.RUnlock()
299
300	if err := b.PostIndex.Delete(post.Id); err != nil {
301		return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, err.Error(), http.StatusInternalServerError)
302	}
303	return nil
304}
305
306func (b *BleveEngine) IndexChannel(channel *model.Channel) *model.AppError {
307	b.Mutex.RLock()
308	defer b.Mutex.RUnlock()
309
310	blvChannel := BLVChannelFromChannel(channel)
311	if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil {
312		return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, err.Error(), http.StatusInternalServerError)
313	}
314	return nil
315}
316
317func (b *BleveEngine) SearchChannels(teamId, term string) ([]string, *model.AppError) {
318	teamIdQ := bleve.NewTermQuery(teamId)
319	teamIdQ.SetField("TeamId")
320	queries := []query.Query{teamIdQ}
321
322	if term != "" {
323		nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term))
324		nameSuggestQ.SetField("NameSuggest")
325		queries = append(queries, nameSuggestQ)
326	}
327
328	query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...))
329	query.Size = model.ChannelSearchDefaultLimit
330	results, err := b.ChannelIndex.Search(query)
331	if err != nil {
332		return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, err.Error(), http.StatusInternalServerError)
333	}
334
335	channelIds := []string{}
336	for _, result := range results.Hits {
337		channelIds = append(channelIds, result.ID)
338	}
339
340	return channelIds, nil
341}
342
343func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError {
344	b.Mutex.RLock()
345	defer b.Mutex.RUnlock()
346
347	if err := b.ChannelIndex.Delete(channel.Id); err != nil {
348		return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, err.Error(), http.StatusInternalServerError)
349	}
350	return nil
351}
352
353func (b *BleveEngine) IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError {
354	b.Mutex.RLock()
355	defer b.Mutex.RUnlock()
356
357	blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds)
358	if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil {
359		return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, err.Error(), http.StatusInternalServerError)
360	}
361	return nil
362}
363
364func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
365	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
366		return []string{}, []string{}, nil
367	}
368
369	// users in channel
370	var queries []query.Query
371	if term != "" {
372		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
373		if options.AllowFullNames {
374			termQ.SetField("SuggestionsWithFullname")
375		} else {
376			termQ.SetField("SuggestionsWithoutFullname")
377		}
378		queries = append(queries, termQ)
379	}
380
381	channelIdQ := bleve.NewTermQuery(channelId)
382	channelIdQ.SetField("ChannelsIds")
383	queries = append(queries, channelIdQ)
384
385	query := bleve.NewConjunctionQuery(queries...)
386
387	uchanSearch := bleve.NewSearchRequest(query)
388	uchanSearch.Size = options.Limit
389	uchan, err := b.UserIndex.Search(uchanSearch)
390	if err != nil {
391		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, err.Error(), http.StatusInternalServerError)
392	}
393
394	// users not in channel
395	boolQ := bleve.NewBooleanQuery()
396
397	if term != "" {
398		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
399		if options.AllowFullNames {
400			termQ.SetField("SuggestionsWithFullname")
401		} else {
402			termQ.SetField("SuggestionsWithoutFullname")
403		}
404		boolQ.AddMust(termQ)
405	}
406
407	teamIdQ := bleve.NewTermQuery(teamId)
408	teamIdQ.SetField("TeamsIds")
409	boolQ.AddMust(teamIdQ)
410
411	outsideChannelIdQ := bleve.NewTermQuery(channelId)
412	outsideChannelIdQ.SetField("ChannelsIds")
413	boolQ.AddMustNot(outsideChannelIdQ)
414
415	if len(restrictedToChannels) > 0 {
416		restrictedChannelsQ := bleve.NewDisjunctionQuery()
417		for _, channelId := range restrictedToChannels {
418			restrictedChannelQ := bleve.NewTermQuery(channelId)
419			restrictedChannelsQ.AddQuery(restrictedChannelQ)
420		}
421		boolQ.AddMust(restrictedChannelsQ)
422	}
423
424	nuchanSearch := bleve.NewSearchRequest(boolQ)
425	nuchanSearch.Size = options.Limit
426	nuchan, err := b.UserIndex.Search(nuchanSearch)
427	if err != nil {
428		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, err.Error(), http.StatusInternalServerError)
429	}
430
431	uchanIds := []string{}
432	for _, result := range uchan.Hits {
433		uchanIds = append(uchanIds, result.ID)
434	}
435
436	nuchanIds := []string{}
437	for _, result := range nuchan.Hits {
438		nuchanIds = append(nuchanIds, result.ID)
439	}
440
441	return uchanIds, nuchanIds, nil
442}
443
444func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
445	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
446		return []string{}, nil
447	}
448
449	var rootQ query.Query
450	if term == "" && teamId == "" && restrictedToChannels == nil {
451		rootQ = bleve.NewMatchAllQuery()
452	} else {
453		boolQ := bleve.NewBooleanQuery()
454
455		if term != "" {
456			termQ := bleve.NewPrefixQuery(strings.ToLower(term))
457			if options.AllowFullNames {
458				termQ.SetField("SuggestionsWithFullname")
459			} else {
460				termQ.SetField("SuggestionsWithoutFullname")
461			}
462			boolQ.AddMust(termQ)
463		}
464
465		if len(restrictedToChannels) > 0 {
466			// restricted channels are already filtered by team, so we
467			// can search only those matches
468			restrictedChannelsQ := []query.Query{}
469			for _, channelId := range restrictedToChannels {
470				channelIdQ := bleve.NewTermQuery(channelId)
471				channelIdQ.SetField("ChannelsIds")
472				restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ)
473			}
474			boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...))
475		} else {
476			// this means that we only need to restrict by team
477			if teamId != "" {
478				teamIdQ := bleve.NewTermQuery(teamId)
479				teamIdQ.SetField("TeamsIds")
480				boolQ.AddMust(teamIdQ)
481			}
482		}
483
484		rootQ = boolQ
485	}
486
487	search := bleve.NewSearchRequest(rootQ)
488	search.Size = options.Limit
489	results, err := b.UserIndex.Search(search)
490	if err != nil {
491		return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, err.Error(), http.StatusInternalServerError)
492	}
493
494	usersIds := []string{}
495	for _, r := range results.Hits {
496		usersIds = append(usersIds, r.ID)
497	}
498
499	return usersIds, nil
500}
501
502func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
503	b.Mutex.RLock()
504	defer b.Mutex.RUnlock()
505
506	if err := b.UserIndex.Delete(user.Id); err != nil {
507		return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, err.Error(), http.StatusInternalServerError)
508	}
509	return nil
510}
511
512func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
513	b.Mutex.RLock()
514	defer b.Mutex.RUnlock()
515
516	blvFile := BLVFileFromFileInfo(file, channelId)
517	if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
518		return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, err.Error(), http.StatusInternalServerError)
519	}
520	return nil
521}
522
523func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
524	channelQueries := []query.Query{}
525	for _, channel := range channels {
526		channelIdQ := bleve.NewTermQuery(channel.Id)
527		channelIdQ.SetField("ChannelId")
528		channelQueries = append(channelQueries, channelIdQ)
529	}
530	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
531
532	var termQueries []query.Query
533	var notTermQueries []query.Query
534	var filters []query.Query
535	var notFilters []query.Query
536
537	for i, params := range searchParams {
538		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
539		if searchParams[0].OrTerms {
540			termOperator = query.MatchQueryOperatorOr
541		}
542
543		// Date, channels and FromUsers filters come in all
544		// searchParams iteration, and as they are global to the
545		// query, we only need to process them once
546		if i == 0 {
547			if len(params.InChannels) > 0 {
548				inChannels := []query.Query{}
549				for _, channelId := range params.InChannels {
550					channelQ := bleve.NewTermQuery(channelId)
551					channelQ.SetField("ChannelId")
552					inChannels = append(inChannels, channelQ)
553				}
554				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
555			}
556
557			if len(params.ExcludedChannels) > 0 {
558				excludedChannels := []query.Query{}
559				for _, channelId := range params.ExcludedChannels {
560					channelQ := bleve.NewTermQuery(channelId)
561					channelQ.SetField("ChannelId")
562					excludedChannels = append(excludedChannels, channelQ)
563				}
564				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
565			}
566
567			if len(params.FromUsers) > 0 {
568				fromUsers := []query.Query{}
569				for _, userId := range params.FromUsers {
570					userQ := bleve.NewTermQuery(userId)
571					userQ.SetField("CreatorId")
572					fromUsers = append(fromUsers, userQ)
573				}
574				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
575			}
576
577			if len(params.ExcludedUsers) > 0 {
578				excludedUsers := []query.Query{}
579				for _, userId := range params.ExcludedUsers {
580					userQ := bleve.NewTermQuery(userId)
581					userQ.SetField("CreatorId")
582					excludedUsers = append(excludedUsers, userQ)
583				}
584				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
585			}
586
587			if len(params.Extensions) > 0 {
588				extensions := []query.Query{}
589				for _, extension := range params.Extensions {
590					extensionQ := bleve.NewTermQuery(extension)
591					extensionQ.SetField("Extension")
592					extensions = append(extensions, extensionQ)
593				}
594				filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
595			}
596
597			if len(params.ExcludedExtensions) > 0 {
598				excludedExtensions := []query.Query{}
599				for _, extension := range params.ExcludedExtensions {
600					extensionQ := bleve.NewTermQuery(extension)
601					extensionQ.SetField("Extension")
602					excludedExtensions = append(excludedExtensions, extensionQ)
603				}
604				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
605			}
606
607			if params.OnDate != "" {
608				before, after := params.GetOnDateMillis()
609				beforeFloat64 := float64(before)
610				afterFloat64 := float64(after)
611				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
612				onDateQ.SetField("CreateAt")
613				filters = append(filters, onDateQ)
614			} else {
615				if params.AfterDate != "" || params.BeforeDate != "" {
616					var min, max *float64
617					if params.AfterDate != "" {
618						minf := float64(params.GetAfterDateMillis())
619						min = &minf
620					}
621
622					if params.BeforeDate != "" {
623						maxf := float64(params.GetBeforeDateMillis())
624						max = &maxf
625					}
626
627					dateQ := bleve.NewNumericRangeQuery(min, max)
628					dateQ.SetField("CreateAt")
629					filters = append(filters, dateQ)
630				}
631
632				if params.ExcludedAfterDate != "" {
633					minf := float64(params.GetExcludedAfterDateMillis())
634					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
635					dateQ.SetField("CreateAt")
636					notFilters = append(notFilters, dateQ)
637				}
638
639				if params.ExcludedBeforeDate != "" {
640					maxf := float64(params.GetExcludedBeforeDateMillis())
641					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
642					dateQ.SetField("CreateAt")
643					notFilters = append(notFilters, dateQ)
644				}
645
646				if params.ExcludedDate != "" {
647					before, after := params.GetExcludedDateMillis()
648					beforef := float64(before)
649					afterf := float64(after)
650					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
651					onDateQ.SetField("CreateAt")
652					notFilters = append(notFilters, onDateQ)
653				}
654			}
655		}
656
657		if params.Terms != "" {
658			terms := []string{}
659			for _, term := range strings.Split(params.Terms, " ") {
660				if strings.HasSuffix(term, "*") {
661					nameQ := bleve.NewWildcardQuery(term)
662					nameQ.SetField("Name")
663					contentQ := bleve.NewWildcardQuery(term)
664					contentQ.SetField("Content")
665					termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
666				} else {
667					terms = append(terms, term)
668				}
669			}
670
671			if len(terms) > 0 {
672				nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
673				nameQ.SetField("Name")
674				nameQ.SetOperator(termOperator)
675				contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
676				contentQ.SetField("Content")
677				contentQ.SetOperator(termOperator)
678				termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
679			}
680		}
681
682		if params.ExcludedTerms != "" {
683			nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
684			nameQ.SetField("Name")
685			nameQ.SetOperator(termOperator)
686			contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
687			contentQ.SetField("Content")
688			contentQ.SetOperator(termOperator)
689			notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
690		}
691	}
692
693	allTermsQ := bleve.NewBooleanQuery()
694	allTermsQ.AddMustNot(notTermQueries...)
695	if searchParams[0].OrTerms {
696		allTermsQ.AddShould(termQueries...)
697	} else {
698		allTermsQ.AddMust(termQueries...)
699	}
700
701	query := bleve.NewBooleanQuery()
702	query.AddMust(channelDisjunctionQ)
703
704	if len(termQueries) > 0 || len(notTermQueries) > 0 {
705		query.AddMust(allTermsQ)
706	}
707
708	if len(filters) > 0 {
709		query.AddMust(bleve.NewConjunctionQuery(filters...))
710	}
711	if len(notFilters) > 0 {
712		query.AddMustNot(notFilters...)
713	}
714
715	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
716	search.SortBy([]string{"-CreateAt"})
717	results, err := b.FileIndex.Search(search)
718	if err != nil {
719		return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, err.Error(), http.StatusInternalServerError)
720	}
721
722	fileIds := []string{}
723
724	for _, r := range results.Hits {
725		fileIds = append(fileIds, r.ID)
726	}
727
728	return fileIds, nil
729}
730
731func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
732	b.Mutex.RLock()
733	defer b.Mutex.RUnlock()
734
735	if err := b.FileIndex.Delete(fileID); err != nil {
736		return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, err.Error(), http.StatusInternalServerError)
737	}
738	return nil
739}
740
741func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
742	resultsCount := int64(0)
743
744	for {
745		// As we are deleting the files after fetching them, we need to keep
746		// From fixed always to 0
747		searchRequest.From = 0
748		searchRequest.Size = batchSize
749		results, err := b.FileIndex.Search(searchRequest)
750		if err != nil {
751			return -1, err
752		}
753		batch := b.FileIndex.NewBatch()
754		for _, file := range results.Hits {
755			batch.Delete(file.ID)
756		}
757		if err := b.FileIndex.Batch(batch); err != nil {
758			return -1, err
759		}
760		resultsCount += int64(results.Hits.Len())
761		if results.Hits.Len() < batchSize {
762			break
763		}
764	}
765
766	return resultsCount, nil
767}
768
769func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError {
770	b.Mutex.RLock()
771	defer b.Mutex.RUnlock()
772
773	query := bleve.NewTermQuery(userID)
774	query.SetField("CreatorId")
775	search := bleve.NewSearchRequest(query)
776	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
777	if err != nil {
778		return model.NewAppError("Bleveengine.DeleteUserFiles",
779			"bleveengine.delete_user_files.error", nil,
780			err.Error(), http.StatusInternalServerError)
781	}
782
783	mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
784
785	return nil
786}
787
788func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError {
789	b.Mutex.RLock()
790	defer b.Mutex.RUnlock()
791
792	query := bleve.NewTermQuery(postID)
793	query.SetField("PostId")
794	search := bleve.NewSearchRequest(query)
795	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
796	if err != nil {
797		return model.NewAppError("Bleveengine.DeletePostFiles",
798			"bleveengine.delete_post_files.error", nil,
799			err.Error(), http.StatusInternalServerError)
800	}
801
802	mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted))
803
804	return nil
805}
806
807func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError {
808	b.Mutex.RLock()
809	defer b.Mutex.RUnlock()
810
811	endTimeFloat := float64(endTime)
812	query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
813	query.SetField("CreateAt")
814	search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
815	search.SortBy([]string{"-CreateAt"})
816
817	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
818	if err != nil {
819		return model.NewAppError("Bleveengine.DeleteFilesBatch",
820			"bleveengine.delete_files_batch.error", nil,
821			err.Error(), http.StatusInternalServerError)
822	}
823
824	mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted))
825
826	return nil
827}
828