1# COPYRIGHT (C) 2020-2021 Nicotine+ Team 2# COPYRIGHT (C) 2016-2018 Mutnick <mutnick@techie.com> 3# COPYRIGHT (C) 2016-2017 Michael Labouebe <gfarmerfr@free.fr> 4# COPYRIGHT (C) 2008-2011 Quinox <quinox@users.sf.net> 5# COPYRIGHT (C) 2006-2009 Daelstorm <daelstorm@gmail.com> 6# COPYRIGHT (C) 2003-2004 Hyriand <hyriand@thegraveyard.org> 7# 8# GNU GENERAL PUBLIC LICENSE 9# Version 3, 29 June 2007 10# 11# This program is free software: you can redistribute it and/or modify 12# it under the terms of the GNU General Public License as published by 13# the Free Software Foundation, either version 3 of the License, or 14# (at your option) any later version. 15# 16# This program is distributed in the hope that it will be useful, 17# but WITHOUT ANY WARRANTY; without even the implied warranty of 18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19# GNU General Public License for more details. 20# 21# You should have received a copy of the GNU General Public License 22# along with this program. If not, see <http://www.gnu.org/licenses/>. 23 24import random 25 26from itertools import islice 27 28from pynicotine import slskmessages 29from pynicotine.logfacility import log 30from pynicotine.utils import PUNCTUATION 31 32 33class Search: 34 35 def __init__(self, core, config, queue, share_dbs, geoip, ui_callback=None): 36 37 self.core = core 38 self.config = config 39 self.queue = queue 40 self.ui_callback = None 41 self.searches = {} 42 self.searchid = int(random.random() * (2 ** 31 - 1)) 43 self.wishlist_interval = 0 44 self.share_dbs = share_dbs 45 self.geoip = geoip 46 self.translatepunctuation = str.maketrans(dict.fromkeys(PUNCTUATION, ' ')) 47 48 # Create wishlist searches 49 for term in config.sections["server"]["autosearch"]: 50 search_id = self.increment_search_id() 51 self.searches[search_id] = {"id": search_id, "term": term, "mode": "wishlist", "ignore": True} 52 53 if hasattr(ui_callback, "search"): 54 self.ui_callback = ui_callback.search 55 56 def server_disconnect(self): 57 58 self.wishlist_interval = 0 59 60 if self.ui_callback: 61 self.ui_callback.server_disconnect() 62 63 def request_folder_download(self, user, folder, visible_files): 64 65 # First queue the visible search results 66 if self.config.sections["transfers"]["reverseorder"]: 67 visible_files.sort(key=lambda x: x[1], reverse=True) 68 69 for file in visible_files: 70 user, fullpath, destination, size, bitrate, length = file 71 72 self.core.transfers.get_file( 73 user, fullpath, destination, 74 size=size, bitrate=bitrate, length=length) 75 76 # Ask for the rest of the files in the folder 77 self.core.transfers.get_folder(user, folder) 78 79 """ Outgoing search requests """ 80 81 @staticmethod 82 def add_allowed_search_id(search_id): 83 """ Allow parsing search result messages for a search ID """ 84 slskmessages.SEARCH_TOKENS_ALLOWED.add(search_id) 85 86 @staticmethod 87 def remove_allowed_search_id(search_id): 88 """ Disallow parsing search result messages for a search ID """ 89 slskmessages.SEARCH_TOKENS_ALLOWED.discard(search_id) 90 91 def add_search(self, search_id, term, mode, ignore): 92 self.searches[search_id] = {"id": search_id, "term": term, "mode": mode, "ignore": ignore} 93 self.add_allowed_search_id(search_id) 94 95 def remove_search(self, search_id): 96 97 self.remove_allowed_search_id(search_id) 98 search = self.searches.get(search_id) 99 100 if search is None: 101 return 102 103 if search["term"] in self.config.sections["server"]["autosearch"]: 104 search["ignore"] = True 105 return 106 107 del self.searches[search_id] 108 109 def process_search_term(self, text, mode, room, user): 110 111 users = [] 112 feedback = None 113 114 if mode == "global": 115 if self.core: 116 feedback = self.core.pluginhandler.outgoing_global_search_event(text) 117 118 if feedback is not None: 119 text = feedback[0] 120 121 elif mode == "rooms": 122 if self.core: 123 feedback = self.core.pluginhandler.outgoing_room_search_event(room, text) 124 125 if feedback is not None: 126 room, text = feedback 127 128 elif mode == "buddies" and self.core: 129 feedback = self.core.pluginhandler.outgoing_buddy_search_event(text) 130 131 if feedback is not None: 132 text = feedback[0] 133 134 elif mode == "user": 135 if user: 136 users = [user] 137 else: 138 return None 139 140 if self.core: 141 feedback = self.core.pluginhandler.outgoing_user_search_event(users, text) 142 143 if feedback is not None: 144 users, text = feedback 145 146 else: 147 log.add("Unknown search mode, not using plugin system. Fix me!") 148 149 return text, room, users 150 151 def do_search(self, text, mode, room=None, user=None): 152 153 # Validate search term and run it through plugins 154 processed_search = self.process_search_term(text, mode, room, user) 155 156 if not processed_search: 157 return None 158 159 text, room, users = processed_search 160 161 # Get a new search ID 162 self.increment_search_id() 163 164 # Get excluded words (starting with "-") 165 searchterm_words = text.split() 166 searchterm_words_special = (p for p in searchterm_words if p.startswith(('-', '*')) and len(p) > 1) 167 168 # Remove words starting with "-", results containing these are excluded by us later 169 searchterm_without_special = ' '.join(p for p in searchterm_words if not p.startswith(('-', '*'))) 170 171 if self.config.sections["searches"]["remove_special_chars"]: 172 """ 173 Remove special characters from search term 174 SoulseekQt doesn't seem to send search results if special characters are included (July 7, 2020) 175 """ 176 stripped_searchterm = ' '.join(searchterm_without_special.translate(self.translatepunctuation).split()) 177 178 # Only modify search term if string also contains non-special characters 179 if stripped_searchterm: 180 searchterm_without_special = stripped_searchterm 181 182 # Remove trailing whitespace 183 searchterm = searchterm_without_special.strip() 184 185 # Append excluded words 186 for word in searchterm_words_special: 187 searchterm += " " + word 188 189 if self.config.sections["searches"]["enable_history"]: 190 items = self.config.sections["searches"]["history"] 191 192 if searchterm in items: 193 items.remove(searchterm) 194 195 items.insert(0, searchterm) 196 197 # Clear old items 198 del items[200:] 199 self.config.write_configuration() 200 201 if mode == "global": 202 self.do_global_search(self.searchid, searchterm) 203 204 elif mode == "rooms": 205 self.do_rooms_search(self.searchid, searchterm, room) 206 207 elif mode == "buddies": 208 self.do_buddies_search(self.searchid, searchterm) 209 210 elif mode == "user": 211 self.do_peer_search(self.searchid, searchterm, users) 212 213 self.add_search(self.searchid, searchterm, mode, ignore=False) 214 215 if self.ui_callback: 216 self.ui_callback.do_search(self.searchid, searchterm, mode, room, user) 217 218 return (self.searchid, searchterm, searchterm_without_special) 219 220 def do_global_search(self, search_id, text): 221 self.queue.append(slskmessages.FileSearch(search_id, text)) 222 223 """ Request a list of related searches from the server. 224 Seemingly non-functional since 2018 (always receiving empty lists). """ 225 226 # self.queue.append(slskmessages.RelatedSearch(text)) 227 228 def do_rooms_search(self, search_id, text, room=None): 229 230 if room != _("Joined Rooms "): 231 self.queue.append(slskmessages.RoomSearch(room, search_id, text)) 232 233 elif self.core.chatrooms.ui_callback is not None: 234 for joined_room in self.core.chatrooms.ui_callback.pages: 235 self.queue.append(slskmessages.RoomSearch(joined_room, search_id, text)) 236 237 def do_buddies_search(self, search_id, text): 238 239 for row in self.config.sections["server"]["userlist"]: 240 if row and isinstance(row, list): 241 user = str(row[0]) 242 self.queue.append(slskmessages.UserSearch(user, search_id, text)) 243 244 def do_peer_search(self, search_id, text, users): 245 for user in users: 246 self.queue.append(slskmessages.UserSearch(user, search_id, text)) 247 248 def do_wishlist_search(self, search_id, text): 249 self.add_allowed_search_id(search_id) 250 self.queue.append(slskmessages.WishlistSearch(search_id, text)) 251 252 def do_wishlist_search_interval(self): 253 254 if self.wishlist_interval == 0: 255 log.add(_("Server does not permit performing wishlist searches at this time")) 256 return False 257 258 searches = self.config.sections["server"]["autosearch"] 259 260 if not searches: 261 return True 262 263 # Search for a maximum of 1 item at each search interval 264 term = searches.pop() 265 searches.insert(0, term) 266 267 for search in self.searches.values(): 268 if search["term"] == term and search["mode"] == "wishlist": 269 search["ignore"] = False 270 self.do_wishlist_search(search["id"], term) 271 break 272 273 return True 274 275 def get_current_search_id(self): 276 return self.searchid 277 278 def increment_search_id(self): 279 self.searchid += 1 280 return self.searchid 281 282 def add_wish(self, wish): 283 284 if not wish: 285 return 286 287 # Get a new search ID 288 self.increment_search_id() 289 290 if wish not in self.config.sections["server"]["autosearch"]: 291 self.config.sections["server"]["autosearch"].append(wish) 292 293 self.add_search(self.searchid, wish, "wishlist", ignore=True) 294 295 if self.ui_callback: 296 self.ui_callback.add_wish(wish) 297 298 def remove_wish(self, wish): 299 300 if wish in self.config.sections["server"]["autosearch"]: 301 self.config.sections["server"]["autosearch"].remove(wish) 302 303 for search in self.searches.values(): 304 if search["term"] == wish and search["mode"] == "wishlist": 305 del search 306 break 307 308 if self.ui_callback: 309 self.ui_callback.remove_wish(wish) 310 311 def is_wish(self, wish): 312 return wish in self.config.sections["server"]["autosearch"] 313 314 def set_wishlist_interval(self, msg): 315 316 self.wishlist_interval = msg.seconds 317 318 if self.ui_callback: 319 self.ui_callback.set_wishlist_interval(msg) 320 321 log.add_search(_("Wishlist wait period set to %s seconds"), msg.seconds) 322 323 def file_search_result(self, msg): 324 325 if not self.ui_callback or msg.token not in slskmessages.SEARCH_TOKENS_ALLOWED: 326 return 327 328 search = self.searches.get(msg.token) 329 330 if search is None or search["ignore"]: 331 return 332 333 conn = msg.conn 334 username = conn.init.target_user 335 addr = conn.addr 336 337 if addr: 338 country = self.geoip.get_country_code(addr[0]) 339 else: 340 country = "" 341 342 if country == "-": 343 country = "" 344 345 self.ui_callback.show_search_result(msg, username, country) 346 347 """ Incoming search requests """ 348 349 @staticmethod 350 def update_search_results(results, word_indices, exclude_word=False): 351 """ Updates the search result list with indices for a new word """ 352 353 if word_indices is None: 354 if exclude_word: 355 # We don't care if an excluded word doesn't exist in our DB 356 return results 357 358 # Included word does not exist in our DB, no results 359 return None 360 361 if results is None: 362 if exclude_word: 363 # No results yet, but word is excluded. Bail. 364 return set() 365 366 # First match for included word, return results 367 return set(word_indices) 368 369 if exclude_word: 370 # Remove results for excluded word 371 results.difference_update(word_indices) 372 else: 373 # Only retain common results for all words so far 374 results.intersection_update(word_indices) 375 376 return results 377 378 def create_search_result_list(self, searchterm, wordindex, excluded_words, partial_words): 379 """ Returns a list of common file indices for each word in a search term """ 380 381 try: 382 words = searchterm.split() 383 original_length = len(words) 384 results = None 385 i = 0 386 387 while i < len(words): 388 word = words[i] 389 exclude_word = False 390 i += 1 391 392 if word in excluded_words: 393 # Excluded search words (e.g. -hello) 394 395 if results is None and i < original_length: 396 # Re-append the word so we can re-process it once we've found a match 397 words.append(word) 398 continue 399 400 exclude_word = True 401 402 elif word in partial_words: 403 # Partial search words (e.g. *ello) 404 405 partial_results = set() 406 407 for complete_word, indices in wordindex.items(): 408 if complete_word.endswith(word): 409 partial_results.update(indices) 410 411 if partial_results: 412 results = self.update_search_results(results, partial_results) 413 continue 414 415 results = self.update_search_results(results, wordindex.get(word), exclude_word) 416 417 if results is None: 418 # No matches found 419 break 420 421 return results 422 423 except ValueError: 424 # DB is closed, perhaps due to rescanning shares or closing the application 425 return None 426 427 def process_search_request(self, searchterm, user, searchid, direct=False): 428 """ Note: since this section is accessed every time a search request arrives, 429 several times a second, please keep it as optimized and memory 430 sparse as possible! """ 431 432 if not searchterm: 433 return 434 435 if not self.config.sections["searches"]["search_results"]: 436 # Don't return _any_ results when this option is disabled 437 return 438 439 if not direct and user == self.config.sections["server"]["login"]: 440 # We shouldn't send a search response if we initiated the search request, 441 # unless we're specifically searching our own username 442 return 443 444 maxresults = self.config.sections["searches"]["maxresults"] 445 446 if maxresults == 0: 447 return 448 449 # Remember excluded/partial words for later 450 excluded_words = [] 451 partial_words = [] 452 453 if '-' in searchterm or '*' in searchterm: 454 for word in searchterm.split(): 455 if len(word) < 1: 456 continue 457 458 if word.startswith('-'): 459 for subword in word.translate(self.translatepunctuation).split(): 460 excluded_words.append(subword) 461 462 elif word.startswith('*'): 463 for subword in word.translate(self.translatepunctuation).split(): 464 partial_words.append(subword) 465 466 # Strip punctuation 467 searchterm_old = searchterm 468 searchterm = searchterm.lower().translate(self.translatepunctuation).strip() 469 470 if len(searchterm) < self.config.sections["searches"]["min_search_chars"]: 471 # Don't send search response if search term contains too few characters 472 return 473 474 checkuser, _reason = self.core.network_filter.check_user(user, None) 475 476 if not checkuser: 477 return 478 479 if checkuser == 2: 480 wordindex = self.share_dbs.get("buddywordindex") 481 else: 482 wordindex = self.share_dbs.get("wordindex") 483 484 if wordindex is None: 485 return 486 487 # Find common file matches for each word in search term 488 resultlist = self.create_search_result_list(searchterm, wordindex, excluded_words, partial_words) 489 490 if not resultlist: 491 return 492 493 if checkuser == 2: 494 fileindex = self.share_dbs.get("buddyfileindex") 495 else: 496 fileindex = self.share_dbs.get("fileindex") 497 498 if fileindex is None: 499 return 500 501 fileinfos = [] 502 numresults = min(len(resultlist), maxresults) 503 504 for index in islice(resultlist, numresults): 505 fileinfo = fileindex.get(repr(index)) 506 507 if fileinfo is not None: 508 fileinfos.append(fileinfo) 509 510 if numresults != len(fileinfos): 511 log.add_debug(("File index inconsistency while responding to search request " 512 "\"%(query)s\". %(expected_num)s results expected, but only %(total_num)s " 513 "results were found in database."), { 514 "query": searchterm_old, 515 "expected_num": numresults, 516 "total_num": len(fileinfos) 517 }) 518 519 numresults = len(fileinfos) 520 521 if not numresults: 522 return 523 524 uploadspeed = self.core.transfers.upload_speed 525 queuesize = self.core.transfers.get_upload_queue_size() 526 slotsavail = self.core.transfers.allow_new_uploads() 527 fifoqueue = self.config.sections["transfers"]["fifoqueue"] 528 529 message = slskmessages.FileSearchResult( 530 None, self.config.sections["server"]["login"], 531 searchid, fileinfos, slotsavail, uploadspeed, queuesize, fifoqueue) 532 533 self.core.send_message_to_peer(user, message) 534 535 if direct: 536 log.add_search( 537 _("User %(user)s is directly searching for \"%(query)s\", returning %(num)i results"), { 538 'user': user, 539 'query': searchterm_old, 540 'num': numresults 541 }) 542 else: 543 log.add_search( 544 _("User %(user)s is searching for \"%(query)s\", returning %(num)i results"), { 545 'user': user, 546 'query': searchterm_old, 547 'num': numresults 548 }) 549