1# Copyright (c) 2014-2021 Cedric Bellegarde <cedric.bellegarde@adishatz.org> 2# Copyright (c) 2019 Jordi Romera <jordiromera@users.sourceforge.net> 3# This program is free software: you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation, either version 3 of the License, or 6# (at your option) any later version. 7# This program is distributed in the hope that it will be useful, 8# but WITHOUT ANY WARRANTY; without even the implied warranty of 9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10# GNU General Public License for more details. 11# You should have received a copy of the GNU General Public License 12# along with this program. If not, see <http://www.gnu.org/licenses/>. 13 14from gi.repository import GLib, GObject, Gio, Gtk 15 16from gi.repository.Gio import FILE_ATTRIBUTE_STANDARD_NAME, \ 17 FILE_ATTRIBUTE_STANDARD_TYPE, \ 18 FILE_ATTRIBUTE_STANDARD_IS_HIDDEN,\ 19 FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,\ 20 FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET,\ 21 FILE_ATTRIBUTE_TIME_MODIFIED,\ 22 FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE 23 24from gettext import gettext as _ 25from time import time, sleep 26from urllib.parse import urlparse 27from multiprocessing import cpu_count 28 29from lollypop.collection_item import CollectionItem 30from lollypop.inotify import Inotify 31from lollypop.define import App, ScanType, Type, StorageType, ScanUpdate 32from lollypop.define import FileType 33from lollypop.sqlcursor import SqlCursor 34from lollypop.tagreader import TagReader, Discoverer 35from lollypop.logger import Logger 36from lollypop.database_history import History 37from lollypop.objects_track import Track 38from lollypop.utils_file import is_audio, is_pls, get_mtime, get_file_type 39from lollypop.utils_album import tracks_to_albums 40from lollypop.utils import emit_signal, profile, split_list 41from lollypop.utils import get_lollypop_album_id, get_lollypop_track_id 42 43 44SCAN_QUERY_INFO = "{},{},{},{},{},{}".format( 45 FILE_ATTRIBUTE_STANDARD_NAME, 46 FILE_ATTRIBUTE_STANDARD_TYPE, 47 FILE_ATTRIBUTE_STANDARD_IS_HIDDEN, 48 FILE_ATTRIBUTE_STANDARD_IS_SYMLINK, 49 FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET, 50 FILE_ATTRIBUTE_TIME_MODIFIED) 51 52 53class CollectionScanner(GObject.GObject, TagReader): 54 """ 55 Scan user music collection 56 """ 57 __gsignals__ = { 58 "scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool,)), 59 "updated": (GObject.SignalFlags.RUN_FIRST, None, 60 (GObject.TYPE_PYOBJECT, int)) 61 } 62 63 def __init__(self): 64 """ 65 Init collection scanner 66 """ 67 GObject.GObject.__init__(self) 68 self.__thread = None 69 self.__tags = {} 70 self.__items = [] 71 self.__notified_ids = [] 72 self.__pending_new_artist_ids = [] 73 self.__history = History() 74 self.__progress_total = 1 75 self.__progress_count = 0 76 self.__progress_fraction = 0 77 self.__disable_compilations = not App().settings.get_value( 78 "show-compilations") 79 if App().settings.get_value("auto-update"): 80 self.__inotify = Inotify() 81 else: 82 self.__inotify = None 83 App().albums.update_max_count() 84 85 def update(self, scan_type, uris=[]): 86 """ 87 Update database 88 @param scan_type as ScanType 89 @param uris as [str] 90 """ 91 self.__disable_compilations = not App().settings.get_value( 92 "show-compilations") 93 App().lookup_action("update_db").set_enabled(False) 94 # Stop previous scan 95 if self.is_locked() and scan_type != ScanType.EXTERNAL: 96 self.stop() 97 GLib.timeout_add(250, self.update, scan_type, uris) 98 return 99 elif App().ws_director.collection_ws is not None and\ 100 not App().ws_director.collection_ws.stop(): 101 GLib.timeout_add(250, self.update, scan_type, uris) 102 return 103 else: 104 if scan_type == ScanType.FULL: 105 uris = App().settings.get_music_uris() 106 if not uris: 107 return 108 # Register to progressbar 109 if scan_type != ScanType.EXTERNAL: 110 App().window.container.progress.add(self) 111 App().window.container.progress.set_fraction(0, self) 112 Logger.info("Scan started") 113 # Launch scan in a separate thread 114 self.__thread = App().task_helper.run(self.__scan, scan_type, uris) 115 116 def save_album(self, item): 117 """ 118 Add album to DB 119 @param item as CollectionItem 120 """ 121 Logger.debug("CollectionScanner::save_album(): " 122 "Add album artists %s" % item.album_artists) 123 (item.new_album_artist_ids, 124 item.album_artist_ids) = self.add_artists(item.album_artists, 125 item.aa_sortnames, 126 item.mb_album_artist_id) 127 # We handle artists already created by any previous save_track() 128 for artist_id in item.album_artist_ids: 129 if artist_id in self.__pending_new_artist_ids: 130 item.new_album_artist_ids.append(artist_id) 131 self.__pending_new_artist_ids.remove(artist_id) 132 133 item.lp_album_id = get_lollypop_album_id(item.album_name, 134 item.album_artists, 135 item.year) 136 Logger.debug("CollectionScanner::save_track(): Add album: " 137 "%s, %s" % (item.album_name, item.album_artist_ids)) 138 (item.new_album, item.album_id) = self.add_album( 139 item.album_name, 140 item.mb_album_id, 141 item.lp_album_id, 142 item.album_artist_ids, 143 item.uri, 144 item.album_loved, 145 item.album_pop, 146 item.album_rate, 147 item.album_synced, 148 item.album_mtime, 149 item.storage_type) 150 if item.year is not None: 151 App().albums.set_year(item.album_id, item.year) 152 App().albums.set_timestamp(item.album_id, item.timestamp) 153 154 def save_track(self, item): 155 """ 156 Add track to DB 157 @param item as CollectionItem 158 """ 159 Logger.debug( 160 "CollectionScanner::save_track(): Add artists %s" % item.artists) 161 (item.new_artist_ids, 162 item.artist_ids) = self.add_artists(item.artists, 163 item.a_sortnames, 164 item.mb_artist_id) 165 166 self.__pending_new_artist_ids += item.new_artist_ids 167 missing_artist_ids = list( 168 set(item.album_artist_ids) - set(item.artist_ids)) 169 # Special case for broken tags 170 # If all artist album tags are missing 171 # Can't do more because don't want to break split album behaviour 172 if len(missing_artist_ids) == len(item.album_artist_ids): 173 item.artist_ids += missing_artist_ids 174 175 if item.genres is None: 176 (item.new_genre_ids, item.genre_ids) = ([], [Type.WEB]) 177 else: 178 (item.new_genre_ids, item.genre_ids) = self.add_genres(item.genres) 179 180 item.lp_track_id = get_lollypop_track_id(item.track_name, 181 item.artists, 182 item.album_name) 183 184 # Add track to db 185 Logger.debug("CollectionScanner::save_track(): Add track") 186 item.track_id = App().tracks.add(item.track_name, 187 item.uri, 188 item.duration, 189 item.tracknumber, 190 item.discnumber, 191 item.discname, 192 item.album_id, 193 item.original_year, 194 item.original_timestamp, 195 item.track_pop, 196 item.track_rate, 197 item.track_loved, 198 item.track_ltime, 199 item.track_mtime, 200 item.mb_track_id, 201 item.lp_track_id, 202 item.bpm, 203 item.storage_type) 204 Logger.debug("CollectionScanner::save_track(): Update track") 205 self.update_track(item) 206 Logger.debug("CollectionScanner::save_track(): Update album") 207 self.update_album(item) 208 209 def update_album(self, item): 210 """ 211 Update album artists based on album-artist and artist tags 212 This code auto handle compilations: empty "album artist" with 213 different artists 214 @param item as CollectionItem 215 """ 216 if item.album_artist_ids: 217 App().albums.set_artist_ids(item.album_id, item.album_artist_ids) 218 # Set artist ids based on content 219 else: 220 if item.compilation: 221 new_album_artist_ids = [Type.COMPILATIONS] 222 else: 223 new_album_artist_ids = App().albums.calculate_artist_ids( 224 item.album_id, self.__disable_compilations) 225 App().albums.set_artist_ids(item.album_id, new_album_artist_ids) 226 # We handle artists already created by any previous save_track() 227 item.new_album_artist_ids = [] 228 for artist_id in new_album_artist_ids: 229 if artist_id in self.__pending_new_artist_ids: 230 item.new_album_artist_ids.append(artist_id) 231 self.__pending_new_artist_ids.remove(artist_id) 232 # Update lp_album_id 233 lp_album_id = get_lollypop_album_id(item.album_name, 234 item.album_artists, 235 item.year) 236 if lp_album_id != item.lp_album_id: 237 App().album_art.move(item.lp_album_id, lp_album_id) 238 App().albums.set_lp_album_id(item.album_id, lp_album_id) 239 item.lp_album_id = lp_album_id 240 # Update album genres 241 for genre_id in item.genre_ids: 242 App().albums.add_genre(item.album_id, genre_id) 243 App().cache.clear_durations(item.album_id) 244 245 def update_track(self, item): 246 """ 247 Set track artists/genres 248 @param item as CollectionItem 249 """ 250 # Set artists/genres for track 251 for artist_id in item.artist_ids: 252 App().tracks.add_artist(item.track_id, artist_id) 253 for genre_id in item.genre_ids: 254 App().tracks.add_genre(item.track_id, genre_id) 255 256 def del_from_db(self, uri, backup): 257 """ 258 Delete track from db 259 @param uri as str 260 @param backup as bool 261 @return (popularity, ltime, mtime, 262 loved album, album_popularity, album_rate) 263 """ 264 try: 265 track_id = App().tracks.get_id_by_uri(uri) 266 duration = App().tracks.get_duration(track_id) 267 album_id = App().tracks.get_album_id(track_id) 268 album_artist_ids = App().albums.get_artist_ids(album_id) 269 artist_ids = App().tracks.get_artist_ids(track_id) 270 track_pop = App().tracks.get_popularity(track_id) 271 track_rate = App().tracks.get_rate(track_id) 272 track_ltime = App().tracks.get_ltime(track_id) 273 album_mtime = App().tracks.get_mtime(track_id) 274 track_loved = App().tracks.get_loved(track_id) 275 album_pop = App().albums.get_popularity(album_id) 276 album_rate = App().albums.get_rate(album_id) 277 album_loved = App().albums.get_loved(album_id) 278 album_synced = App().albums.get_synced(album_id) 279 if backup: 280 f = Gio.File.new_for_uri(uri) 281 name = f.get_basename() 282 self.__history.add(name, duration, track_pop, track_rate, 283 track_ltime, album_mtime, track_loved, 284 album_loved, album_pop, album_rate, 285 album_synced) 286 App().tracks.remove(track_id) 287 genre_ids = App().tracks.get_genre_ids(track_id) 288 App().albums.clean() 289 App().genres.clean() 290 App().artists.clean() 291 App().cache.clear_durations(album_id) 292 SqlCursor.commit(App().db) 293 item = CollectionItem(album_id=album_id) 294 if not App().albums.get_name(album_id): 295 item.artist_ids = [] 296 for artist_id in album_artist_ids + artist_ids: 297 if not App().artists.get_name(artist_id): 298 item.artist_ids.append(artist_id) 299 item.genre_ids = [] 300 for genre_id in genre_ids: 301 if not App().genres.get_name(genre_id): 302 item.genre_ids.append(genre_id) 303 emit_signal(self, "updated", item, ScanUpdate.REMOVED) 304 else: 305 # Force genre for album 306 genre_ids = App().tracks.get_album_genre_ids(album_id) 307 App().albums.set_genre_ids(album_id, genre_ids) 308 emit_signal(self, "updated", item, ScanUpdate.MODIFIED) 309 return (track_pop, track_rate, track_ltime, album_mtime, 310 track_loved, album_loved, album_pop, album_rate) 311 except Exception as e: 312 Logger.error("CollectionScanner::del_from_db: %s" % e) 313 return (0, 0, 0, 0, False, False, 0, 0) 314 315 def is_locked(self): 316 """ 317 True if db locked 318 @return bool 319 """ 320 return self.__thread is not None and self.__thread.is_alive() 321 322 def stop(self): 323 """ 324 Stop scan 325 """ 326 self.__thread = None 327 328 def reset_database(self): 329 """ 330 Reset database 331 """ 332 from lollypop.app_notification import AppNotification 333 App().window.container.progress.add(self) 334 App().window.container.progress.set_fraction(0, self) 335 self.__progress_fraction = 0 336 notification = AppNotification(_("Resetting database"), [], [], 10000) 337 notification.show() 338 App().window.container.add_overlay(notification) 339 notification.set_reveal_child(True) 340 App().task_helper.run(self.__reset_database) 341 342 @property 343 def inotify(self): 344 """ 345 Get Inotify object 346 @return Inotify 347 """ 348 return self.__inotify 349 350####################### 351# PRIVATE # 352####################### 353 def __reset_database(self): 354 """ 355 Reset database 356 """ 357 def update_ui(): 358 App().window.container.go_home() 359 App().scanner.update(ScanType.FULL) 360 App().player.stop() 361 if App().ws_director.collection_ws is not None: 362 App().ws_director.collection_ws.stop() 363 uris = App().tracks.get_uris() 364 i = 0 365 SqlCursor.add(App().db) 366 SqlCursor.add(self.__history) 367 count = len(uris) 368 for uri in uris: 369 self.del_from_db(uri, True) 370 self.__update_progress(i, count, 0.01) 371 i += 1 372 App().tracks.del_persistent(False) 373 App().tracks.clean(False) 374 App().albums.clean(False) 375 App().artists.clean(False) 376 App().genres.clean(False) 377 App().cache.clear_table("duration") 378 SqlCursor.commit(App().db) 379 SqlCursor.remove(App().db) 380 SqlCursor.commit(self.__history) 381 SqlCursor.remove(self.__history) 382 GLib.idle_add(update_ui) 383 384 def __update_progress(self, current, total, allowed_diff): 385 """ 386 Update progress bar status 387 @param current as int 388 @param total as int 389 @param allowed_diff as float => allows to prevent 390 main loop flooding 391 """ 392 new_fraction = current / total 393 if new_fraction > self.__progress_fraction + allowed_diff: 394 self.__progress_fraction = new_fraction 395 GLib.idle_add(App().window.container.progress.set_fraction, 396 new_fraction, self) 397 398 def __finish(self, items): 399 """ 400 Notify from main thread when scan finished 401 @param items as [CollectionItem] 402 """ 403 track_ids = [item.track_id for item in items] 404 self.__thread = None 405 Logger.info("Scan finished") 406 App().lookup_action("update_db").set_enabled(True) 407 App().window.container.progress.set_fraction(1.0, self) 408 self.stop() 409 emit_signal(self, "scan-finished", track_ids) 410 # Update max count value 411 App().albums.update_max_count() 412 # Update featuring 413 App().artists.update_featuring() 414 if App().ws_director.collection_ws is not None: 415 App().ws_director.collection_ws.start() 416 417 def __add_monitor(self, dirs): 418 """ 419 Monitor any change in a list of directory 420 @param dirs as str or list of directory to be monitored 421 """ 422 if self.__inotify is None: 423 return 424 # Add monitors on dirs 425 for d in dirs: 426 # Handle a stop request 427 if self.__thread is None: 428 break 429 if d.startswith("file://"): 430 self.__inotify.add_monitor(d) 431 432 @profile 433 def __get_objects_for_uris(self, scan_type, uris): 434 """ 435 Get all tracks and dirs in uris 436 @param scan_type as ScanType 437 @param uris as string 438 @return ([(int, str)], [str], [str]) 439 ([(mtime, file)], [dir], [stream]) 440 """ 441 files = [] 442 dirs = [] 443 streams = [] 444 walk_uris = [] 445 # Check collection exists 446 for uri in uris: 447 parsed = urlparse(uri) 448 if parsed.scheme in ["http", "https"]: 449 streams.append(uri) 450 else: 451 f = Gio.File.new_for_uri(uri) 452 if f.query_exists(): 453 walk_uris.append(uri) 454 else: 455 return ([], [], []) 456 457 while walk_uris: 458 uri = walk_uris.pop(0) 459 try: 460 # Directly add files, walk through directories 461 f = Gio.File.new_for_uri(uri) 462 info = f.query_info(SCAN_QUERY_INFO, 463 Gio.FileQueryInfoFlags.NONE, 464 None) 465 if info.get_file_type() == Gio.FileType.DIRECTORY: 466 dirs.append(uri) 467 infos = f.enumerate_children(SCAN_QUERY_INFO, 468 Gio.FileQueryInfoFlags.NONE, 469 None) 470 for info in infos: 471 f = infos.get_child(info) 472 child_uri = f.get_uri() 473 if info.get_is_hidden(): 474 continue 475 # User do not want internal symlinks 476 elif info.get_is_symlink() and\ 477 App().settings.get_value("ignore-symlinks"): 478 continue 479 walk_uris.append(child_uri) 480 infos.close(None) 481 # Only happens if files passed as args 482 else: 483 mtime = get_mtime(info) 484 files.append((mtime, uri)) 485 except Exception as e: 486 Logger.error("CollectionScanner::__get_objects_for_uris(): %s" 487 % e) 488 files.sort(reverse=True) 489 return (files, dirs, streams) 490 491 @profile 492 def __scan(self, scan_type, uris): 493 """ 494 Scan music collection for music files 495 @param scan_type as ScanType 496 @param uris as [str] 497 @thread safe 498 """ 499 try: 500 self.__items = [] 501 App().art.clean_rounded() 502 (files, dirs, streams) = self.__get_objects_for_uris( 503 scan_type, uris) 504 if len(uris) != len(streams) and not files: 505 self.__flatpak_migration() 506 App().notify.send("Lollypop", 507 _("Scan disabled, missing collection")) 508 return 509 if scan_type == ScanType.NEW_FILES: 510 db_uris = App().tracks.get_uris(uris) 511 else: 512 db_uris = App().tracks.get_uris() 513 514 # Get mtime of all tracks to detect which has to be updated 515 db_mtimes = App().tracks.get_mtimes() 516 # * 2 => Scan + Save 517 self.__progress_total = len(files) * 2 + len(streams) 518 self.__progress_count = 0 519 self.__progress_fraction = 0 520 # Min: 1 thread, Max: 5 threads 521 count = max(1, min(5, cpu_count() // 2)) 522 split_files = split_list(files, count) 523 self.__tags = {} 524 self.__notified_ids = [] 525 self.__pending_new_artist_ids = [] 526 threads = [] 527 for files in split_files: 528 thread = App().task_helper.run(self.__scan_files, 529 files, db_mtimes, 530 scan_type) 531 threads.append(thread) 532 while threads: 533 sleep(0.1) 534 thread = threads[0] 535 if not thread.is_alive(): 536 threads.remove(thread) 537 538 SqlCursor.add(App().db) 539 if scan_type == ScanType.EXTERNAL: 540 storage_type = StorageType.EXTERNAL 541 else: 542 storage_type = StorageType.COLLECTION 543 self.__items += self.__save_in_db(storage_type) 544 # Add streams to DB, only happening on command line/m3u files 545 self.__items += self.__save_streams_in_db(streams, storage_type) 546 547 self.__remove_old_tracks(db_uris, scan_type) 548 549 if scan_type == ScanType.EXTERNAL: 550 albums = tracks_to_albums( 551 [Track(item.track_id) for item in self.__items]) 552 App().player.play_albums(albums) 553 else: 554 self.__add_monitor(dirs) 555 GLib.idle_add(self.__finish, self.__items) 556 self.__tags = {} 557 self.__items = [] 558 self.__pending_new_artist_ids = [] 559 except Exception as e: 560 Logger.warning("CollectionScanner::__scan(): %s", e) 561 SqlCursor.remove(App().db) 562 App().settings.set_value("flatpak-access-migration", 563 GLib.Variant("b", True)) 564 565 def __scan_to_handle(self, uri): 566 """ 567 Check if file has to be handle by scanner 568 @param f as Gio.File 569 @return bool 570 """ 571 try: 572 file_type = get_file_type(uri) 573 # Get file type using Gio (slower) 574 if file_type == FileType.UNKNOWN: 575 f = Gio.File.new_for_uri(uri) 576 info = f.query_info(FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE, 577 Gio.FileQueryInfoFlags.NONE) 578 if is_pls(info): 579 file_type = FileType.PLS 580 elif is_audio(info): 581 file_type = FileType.AUDIO 582 if file_type == FileType.PLS: 583 Logger.debug("Importing playlist %s" % uri) 584 if App().settings.get_value("import-playlists"): 585 App().playlists.import_tracks(uri) 586 elif file_type == FileType.AUDIO: 587 Logger.debug("Importing audio %s" % uri) 588 return True 589 except Exception as e: 590 Logger.error("CollectionScanner::__scan_to_handle(): %s" % e) 591 return False 592 593 def __scan_files(self, files, db_mtimes, scan_type): 594 """ 595 Scan music collection for new audio files 596 @param files as [str] 597 @param db_mtimes as {} 598 @param scan_type as ScanType 599 @thread safe 600 """ 601 discoverer = Discoverer() 602 try: 603 # Scan new files 604 for (mtime, uri) in files: 605 # Handle a stop request 606 if self.__thread is None and scan_type != ScanType.EXTERNAL: 607 raise Exception("cancelled") 608 try: 609 if not self.__scan_to_handle(uri): 610 self.__progress_count += 2 611 continue 612 db_mtime = db_mtimes.get(uri, 0) 613 if mtime > db_mtime: 614 # Do not use mtime if not intial scan 615 if db_mtimes: 616 mtime = int(time()) 617 self.__tags[uri] = self.__get_tags(discoverer, 618 uri, mtime) 619 self.__progress_count += 1 620 self.__update_progress(self.__progress_count, 621 self.__progress_total, 622 0.001) 623 else: 624 # We want to play files, so put them in items 625 if scan_type == ScanType.EXTERNAL: 626 track_id = App().tracks.get_id_by_uri(uri) 627 item = CollectionItem(track_id=track_id) 628 self.__items.append(item) 629 self.__progress_count += 2 630 self.__update_progress(self.__progress_count, 631 self.__progress_total, 632 0.1) 633 except Exception as e: 634 Logger.error("Scanning file: %s, %s" % (uri, e)) 635 except Exception as e: 636 Logger.warning("CollectionScanner::__scan_files(): % s" % e) 637 638 def __save_in_db(self, storage_type): 639 """ 640 Save current tags into DB 641 @param storage_type as StorageType 642 @return [CollectionItem] 643 """ 644 items = [] 645 for uri in list(self.__tags.keys()): 646 # Handle a stop request 647 if self.__thread is None: 648 raise Exception("cancelled") 649 Logger.debug("Adding file: %s" % uri) 650 tags = self.__tags[uri] 651 item = self.__add2db(uri, *tags, storage_type) 652 items.append(item) 653 self.__progress_count += 1 654 self.__update_progress(self.__progress_count, 655 self.__progress_total, 656 0.001) 657 if item.album_id not in self.__notified_ids: 658 self.__notified_ids.append(item.album_id) 659 self.__notify_ui(item) 660 del self.__tags[uri] 661 # Handle a stop request 662 if self.__thread is None: 663 raise Exception("cancelled") 664 return items 665 666 def __save_streams_in_db(self, streams, storage_type): 667 """ 668 Save http stream to DB 669 @param streams as [str] 670 @param storage_type as StorageType 671 @return [CollectionItem] 672 """ 673 items = [] 674 for uri in streams: 675 parsed = urlparse(uri) 676 item = self.__add2db(uri, parsed.path, parsed.netloc, 677 None, "", "", parsed.netloc, 678 parsed.netloc, "", False, 0, False, 0, 0, 0, 679 None, 0, "", "", "", "", 1, 0, 0, 0, 0, 0, 680 False, 0, False, storage_type) 681 items.append(item) 682 self.__progress_count += 1 683 return items 684 685 def __notify_ui(self, item): 686 """ 687 Notify UI for item 688 @param items as CollectionItem 689 """ 690 SqlCursor.commit(App().db) 691 if item.new_album: 692 emit_signal(self, "updated", item, ScanUpdate.ADDED) 693 else: 694 emit_signal(self, "updated", item, ScanUpdate.MODIFIED) 695 696 def __remove_old_tracks(self, uris, scan_type): 697 """ 698 Remove non existent tracks from DB 699 @param scan_type as ScanType 700 """ 701 if scan_type != ScanType.EXTERNAL and self.__thread is not None: 702 # We need to check files are always in collections 703 if scan_type == ScanType.FULL: 704 collections = App().settings.get_music_uris() 705 else: 706 collections = None 707 for uri in uris: 708 # Handle a stop request 709 if self.__thread is None: 710 raise Exception("cancelled") 711 in_collection = True 712 if collections is not None: 713 in_collection = False 714 for collection in collections: 715 if collection in uri: 716 in_collection = True 717 break 718 f = Gio.File.new_for_uri(uri) 719 if not in_collection: 720 Logger.warning( 721 "Removed, not in collection anymore: %s -> %s", 722 uri, collections) 723 self.del_from_db(uri, True) 724 elif not f.query_exists(): 725 Logger.warning("Removed, file has been deleted: %s", uri) 726 self.del_from_db(uri, True) 727 728 def __get_tags(self, discoverer, uri, track_mtime): 729 """ 730 Read track tags 731 @param discoverer as Discoverer 732 @param uri as string 733 @param track_mtime as int 734 @return () 735 """ 736 f = Gio.File.new_for_uri(uri) 737 info = discoverer.get_info(uri) 738 tags = info.get_tags() 739 name = f.get_basename() 740 duration = int(info.get_duration() / 1000000) 741 Logger.debug("CollectionScanner::add2db(): Restore stats") 742 # Restore stats 743 track_id = App().tracks.get_id_by_uri(uri) 744 if track_id is None: 745 track_id = App().tracks.get_id_by_basename_duration(name, 746 duration) 747 if track_id is None: 748 (track_pop, track_rate, track_ltime, 749 album_mtime, track_loved, album_loved, 750 album_pop, album_rate, album_synced) = self.__history.get( 751 name, duration) 752 # Delete track and restore from it 753 else: 754 (track_pop, track_rate, track_ltime, 755 album_mtime, track_loved, album_loved, 756 album_pop, album_rate) = self.del_from_db(uri, False) 757 758 Logger.debug("CollectionScanner::add2db(): Read tags") 759 title = self.get_title(tags, name) 760 version = self.get_version(tags) 761 if version != "": 762 title += " (%s)" % version 763 artists = self.get_artists(tags) 764 a_sortnames = self.get_artist_sortnames(tags) 765 aa_sortnames = self.get_album_artist_sortnames(tags) 766 album_artists = self.get_album_artists(tags) 767 album_name = self.get_album_name(tags) 768 album_synced = 0 769 mb_album_id = self.get_mb_album_id(tags) 770 mb_track_id = self.get_mb_track_id(tags) 771 mb_artist_id = self.get_mb_artist_id(tags) 772 mb_album_artist_id = self.get_mb_album_artist_id(tags) 773 genres = self.get_genres(tags) 774 discnumber = self.get_discnumber(tags) 775 discname = self.get_discname(tags) 776 tracknumber = self.get_tracknumber(tags, name) 777 # We have popm in tags, override history one 778 tag_track_rate = self.get_popm(tags) 779 if tag_track_rate > 0: 780 track_rate = tag_track_rate 781 if album_mtime == 0: 782 album_mtime = track_mtime 783 bpm = self.get_bpm(tags) 784 compilation = self.get_compilation(tags) 785 (original_year, original_timestamp) = self.get_original_year(tags) 786 (year, timestamp) = self.get_year(tags) 787 if year is None: 788 (year, timestamp) = (original_year, original_timestamp) 789 elif original_year is None: 790 (original_year, original_timestamp) = (year, timestamp) 791 # If no artists tag, use album artist 792 if artists == "": 793 artists = album_artists 794 if App().settings.get_value("import-advanced-artist-tags"): 795 composers = self.get_composers(tags) 796 conductors = self.get_conductors(tags) 797 performers = self.get_performers(tags) 798 remixers = self.get_remixers(tags) 799 artists += ";%s" % performers if performers != "" else "" 800 artists += ";%s" % conductors if conductors != "" else "" 801 artists += ";%s" % composers if composers != "" else "" 802 artists += ";%s" % remixers if remixers != "" else "" 803 if artists == "": 804 artists = _("Unknown") 805 return (title, artists, genres, a_sortnames, aa_sortnames, 806 album_artists, album_name, discname, album_loved, album_mtime, 807 album_synced, album_rate, album_pop, discnumber, year, 808 timestamp, original_year, original_timestamp, 809 mb_album_id, mb_track_id, mb_artist_id, 810 mb_album_artist_id, tracknumber, track_pop, track_rate, bpm, 811 track_mtime, track_ltime, track_loved, duration, compilation) 812 813 def __add2db(self, uri, name, artists, 814 genres, a_sortnames, aa_sortnames, album_artists, album_name, 815 discname, album_loved, album_mtime, album_synced, album_rate, 816 album_pop, discnumber, year, timestamp, 817 original_year, original_timestamp, mb_album_id, 818 mb_track_id, mb_artist_id, mb_album_artist_id, 819 tracknumber, track_pop, track_rate, bpm, track_mtime, 820 track_ltime, track_loved, duration, compilation, 821 storage_type=StorageType.COLLECTION): 822 """ 823 Add new file to DB 824 @param uri as str 825 @param tags as *() 826 @param storage_type as StorageType 827 @return CollectionItem 828 """ 829 item = CollectionItem(uri=uri, 830 track_name=name, 831 artists=artists, 832 genres=genres, 833 a_sortnames=a_sortnames, 834 aa_sortnames=aa_sortnames, 835 album_artists=album_artists, 836 album_name=album_name, 837 discname=discname, 838 album_loved=album_loved, 839 album_mtime=album_mtime, 840 album_synced=album_synced, 841 album_rate=album_rate, 842 album_pop=album_pop, 843 discnumber=discnumber, 844 year=year, 845 timestamp=timestamp, 846 original_year=original_year, 847 original_timestamp=original_timestamp, 848 mb_album_id=mb_album_id, 849 mb_track_id=mb_track_id, 850 mb_artist_id=mb_artist_id, 851 mb_album_artist_id=mb_album_artist_id, 852 tracknumber=tracknumber, 853 track_pop=track_pop, 854 track_rate=track_rate, 855 bpm=bpm, 856 track_mtime=track_mtime, 857 track_ltime=track_ltime, 858 track_loved=track_loved, 859 duration=duration, 860 compilation=compilation, 861 storage_type=storage_type) 862 self.save_album(item) 863 self.save_track(item) 864 return item 865 866 def __flatpak_migration(self): 867 """ 868 https://github.com/flathub/org.gnome.Lollypop/pull/108 869 """ 870 if GLib.file_test("/app", GLib.FileTest.EXISTS) and\ 871 not App().settings.get_value("flatpak-access-migration"): 872 from lollypop.assistant_flatpak import FlatpakAssistant 873 assistant = FlatpakAssistant() 874 assistant.set_position(Gtk.WindowPosition.CENTER_ON_PARENT) 875 assistant.set_transient_for(App().window) 876 GLib.timeout_add(1000, assistant.show) 877