1# Copyright (c) 2014-2021 Cedric Bellegarde <cedric.bellegarde@adishatz.org>
2# Copyright (c) 2019 Jordi Romera <jordiromera@users.sourceforge.net>
3# This program is free software: you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation, either version 3 of the License, or
6# (at your option) any later version.
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU General Public License for more details.
11# You should have received a copy of the GNU General Public License
12# along with this program. If not, see <http://www.gnu.org/licenses/>.
13
14from gi.repository import GLib, GObject, Gio, Gtk
15
16from gi.repository.Gio import FILE_ATTRIBUTE_STANDARD_NAME, \
17                              FILE_ATTRIBUTE_STANDARD_TYPE, \
18                              FILE_ATTRIBUTE_STANDARD_IS_HIDDEN,\
19                              FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,\
20                              FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET,\
21                              FILE_ATTRIBUTE_TIME_MODIFIED,\
22                              FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE
23
24from gettext import gettext as _
25from time import time, sleep
26from urllib.parse import urlparse
27from multiprocessing import cpu_count
28
29from lollypop.collection_item import CollectionItem
30from lollypop.inotify import Inotify
31from lollypop.define import App, ScanType, Type, StorageType, ScanUpdate
32from lollypop.define import FileType
33from lollypop.sqlcursor import SqlCursor
34from lollypop.tagreader import TagReader, Discoverer
35from lollypop.logger import Logger
36from lollypop.database_history import History
37from lollypop.objects_track import Track
38from lollypop.utils_file import is_audio, is_pls, get_mtime, get_file_type
39from lollypop.utils_album import tracks_to_albums
40from lollypop.utils import emit_signal, profile, split_list
41from lollypop.utils import get_lollypop_album_id, get_lollypop_track_id
42
43
44SCAN_QUERY_INFO = "{},{},{},{},{},{}".format(
45                                       FILE_ATTRIBUTE_STANDARD_NAME,
46                                       FILE_ATTRIBUTE_STANDARD_TYPE,
47                                       FILE_ATTRIBUTE_STANDARD_IS_HIDDEN,
48                                       FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,
49                                       FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET,
50                                       FILE_ATTRIBUTE_TIME_MODIFIED)
51
52
53class CollectionScanner(GObject.GObject, TagReader):
54    """
55        Scan user music collection
56    """
57    __gsignals__ = {
58        "scan-finished": (GObject.SignalFlags.RUN_FIRST, None, (bool,)),
59        "updated": (GObject.SignalFlags.RUN_FIRST, None,
60                    (GObject.TYPE_PYOBJECT, int))
61    }
62
63    def __init__(self):
64        """
65            Init collection scanner
66        """
67        GObject.GObject.__init__(self)
68        self.__thread = None
69        self.__tags = {}
70        self.__items = []
71        self.__notified_ids = []
72        self.__pending_new_artist_ids = []
73        self.__history = History()
74        self.__progress_total = 1
75        self.__progress_count = 0
76        self.__progress_fraction = 0
77        self.__disable_compilations = not App().settings.get_value(
78                "show-compilations")
79        if App().settings.get_value("auto-update"):
80            self.__inotify = Inotify()
81        else:
82            self.__inotify = None
83        App().albums.update_max_count()
84
85    def update(self, scan_type, uris=[]):
86        """
87            Update database
88            @param scan_type as ScanType
89            @param uris as [str]
90        """
91        self.__disable_compilations = not App().settings.get_value(
92                "show-compilations")
93        App().lookup_action("update_db").set_enabled(False)
94        # Stop previous scan
95        if self.is_locked() and scan_type != ScanType.EXTERNAL:
96            self.stop()
97            GLib.timeout_add(250, self.update, scan_type, uris)
98            return
99        elif App().ws_director.collection_ws is not None and\
100                not App().ws_director.collection_ws.stop():
101            GLib.timeout_add(250, self.update, scan_type, uris)
102            return
103        else:
104            if scan_type == ScanType.FULL:
105                uris = App().settings.get_music_uris()
106            if not uris:
107                return
108            # Register to progressbar
109            if scan_type != ScanType.EXTERNAL:
110                App().window.container.progress.add(self)
111                App().window.container.progress.set_fraction(0, self)
112            Logger.info("Scan started")
113            # Launch scan in a separate thread
114            self.__thread = App().task_helper.run(self.__scan, scan_type, uris)
115
116    def save_album(self, item):
117        """
118            Add album to DB
119            @param item as CollectionItem
120        """
121        Logger.debug("CollectionScanner::save_album(): "
122                     "Add album artists %s" % item.album_artists)
123        (item.new_album_artist_ids,
124         item.album_artist_ids) = self.add_artists(item.album_artists,
125                                                   item.aa_sortnames,
126                                                   item.mb_album_artist_id)
127        # We handle artists already created by any previous save_track()
128        for artist_id in item.album_artist_ids:
129            if artist_id in self.__pending_new_artist_ids:
130                item.new_album_artist_ids.append(artist_id)
131                self.__pending_new_artist_ids.remove(artist_id)
132
133        item.lp_album_id = get_lollypop_album_id(item.album_name,
134                                                 item.album_artists,
135                                                 item.year)
136        Logger.debug("CollectionScanner::save_track(): Add album: "
137                     "%s, %s" % (item.album_name, item.album_artist_ids))
138        (item.new_album, item.album_id) = self.add_album(
139                                               item.album_name,
140                                               item.mb_album_id,
141                                               item.lp_album_id,
142                                               item.album_artist_ids,
143                                               item.uri,
144                                               item.album_loved,
145                                               item.album_pop,
146                                               item.album_rate,
147                                               item.album_synced,
148                                               item.album_mtime,
149                                               item.storage_type)
150        if item.year is not None:
151            App().albums.set_year(item.album_id, item.year)
152            App().albums.set_timestamp(item.album_id, item.timestamp)
153
154    def save_track(self, item):
155        """
156            Add track to DB
157            @param item as CollectionItem
158        """
159        Logger.debug(
160            "CollectionScanner::save_track(): Add artists %s" % item.artists)
161        (item.new_artist_ids,
162         item.artist_ids) = self.add_artists(item.artists,
163                                             item.a_sortnames,
164                                             item.mb_artist_id)
165
166        self.__pending_new_artist_ids += item.new_artist_ids
167        missing_artist_ids = list(
168            set(item.album_artist_ids) - set(item.artist_ids))
169        # Special case for broken tags
170        # If all artist album tags are missing
171        # Can't do more because don't want to break split album behaviour
172        if len(missing_artist_ids) == len(item.album_artist_ids):
173            item.artist_ids += missing_artist_ids
174
175        if item.genres is None:
176            (item.new_genre_ids, item.genre_ids) = ([], [Type.WEB])
177        else:
178            (item.new_genre_ids, item.genre_ids) = self.add_genres(item.genres)
179
180        item.lp_track_id = get_lollypop_track_id(item.track_name,
181                                                 item.artists,
182                                                 item.album_name)
183
184        # Add track to db
185        Logger.debug("CollectionScanner::save_track(): Add track")
186        item.track_id = App().tracks.add(item.track_name,
187                                         item.uri,
188                                         item.duration,
189                                         item.tracknumber,
190                                         item.discnumber,
191                                         item.discname,
192                                         item.album_id,
193                                         item.original_year,
194                                         item.original_timestamp,
195                                         item.track_pop,
196                                         item.track_rate,
197                                         item.track_loved,
198                                         item.track_ltime,
199                                         item.track_mtime,
200                                         item.mb_track_id,
201                                         item.lp_track_id,
202                                         item.bpm,
203                                         item.storage_type)
204        Logger.debug("CollectionScanner::save_track(): Update track")
205        self.update_track(item)
206        Logger.debug("CollectionScanner::save_track(): Update album")
207        self.update_album(item)
208
209    def update_album(self, item):
210        """
211            Update album artists based on album-artist and artist tags
212            This code auto handle compilations: empty "album artist" with
213            different artists
214            @param item as CollectionItem
215        """
216        if item.album_artist_ids:
217            App().albums.set_artist_ids(item.album_id, item.album_artist_ids)
218        # Set artist ids based on content
219        else:
220            if item.compilation:
221                new_album_artist_ids = [Type.COMPILATIONS]
222            else:
223                new_album_artist_ids = App().albums.calculate_artist_ids(
224                    item.album_id, self.__disable_compilations)
225            App().albums.set_artist_ids(item.album_id, new_album_artist_ids)
226            # We handle artists already created by any previous save_track()
227            item.new_album_artist_ids = []
228            for artist_id in new_album_artist_ids:
229                if artist_id in self.__pending_new_artist_ids:
230                    item.new_album_artist_ids.append(artist_id)
231                    self.__pending_new_artist_ids.remove(artist_id)
232        # Update lp_album_id
233        lp_album_id = get_lollypop_album_id(item.album_name,
234                                            item.album_artists,
235                                            item.year)
236        if lp_album_id != item.lp_album_id:
237            App().album_art.move(item.lp_album_id, lp_album_id)
238            App().albums.set_lp_album_id(item.album_id, lp_album_id)
239            item.lp_album_id = lp_album_id
240        # Update album genres
241        for genre_id in item.genre_ids:
242            App().albums.add_genre(item.album_id, genre_id)
243        App().cache.clear_durations(item.album_id)
244
245    def update_track(self, item):
246        """
247            Set track artists/genres
248            @param item as CollectionItem
249        """
250        # Set artists/genres for track
251        for artist_id in item.artist_ids:
252            App().tracks.add_artist(item.track_id, artist_id)
253        for genre_id in item.genre_ids:
254            App().tracks.add_genre(item.track_id, genre_id)
255
256    def del_from_db(self, uri, backup):
257        """
258            Delete track from db
259            @param uri as str
260            @param backup as bool
261            @return (popularity, ltime, mtime,
262                     loved album, album_popularity, album_rate)
263        """
264        try:
265            track_id = App().tracks.get_id_by_uri(uri)
266            duration = App().tracks.get_duration(track_id)
267            album_id = App().tracks.get_album_id(track_id)
268            album_artist_ids = App().albums.get_artist_ids(album_id)
269            artist_ids = App().tracks.get_artist_ids(track_id)
270            track_pop = App().tracks.get_popularity(track_id)
271            track_rate = App().tracks.get_rate(track_id)
272            track_ltime = App().tracks.get_ltime(track_id)
273            album_mtime = App().tracks.get_mtime(track_id)
274            track_loved = App().tracks.get_loved(track_id)
275            album_pop = App().albums.get_popularity(album_id)
276            album_rate = App().albums.get_rate(album_id)
277            album_loved = App().albums.get_loved(album_id)
278            album_synced = App().albums.get_synced(album_id)
279            if backup:
280                f = Gio.File.new_for_uri(uri)
281                name = f.get_basename()
282                self.__history.add(name, duration, track_pop, track_rate,
283                                   track_ltime, album_mtime, track_loved,
284                                   album_loved, album_pop, album_rate,
285                                   album_synced)
286            App().tracks.remove(track_id)
287            genre_ids = App().tracks.get_genre_ids(track_id)
288            App().albums.clean()
289            App().genres.clean()
290            App().artists.clean()
291            App().cache.clear_durations(album_id)
292            SqlCursor.commit(App().db)
293            item = CollectionItem(album_id=album_id)
294            if not App().albums.get_name(album_id):
295                item.artist_ids = []
296                for artist_id in album_artist_ids + artist_ids:
297                    if not App().artists.get_name(artist_id):
298                        item.artist_ids.append(artist_id)
299                item.genre_ids = []
300                for genre_id in genre_ids:
301                    if not App().genres.get_name(genre_id):
302                        item.genre_ids.append(genre_id)
303                emit_signal(self, "updated", item, ScanUpdate.REMOVED)
304            else:
305                # Force genre for album
306                genre_ids = App().tracks.get_album_genre_ids(album_id)
307                App().albums.set_genre_ids(album_id, genre_ids)
308                emit_signal(self, "updated", item, ScanUpdate.MODIFIED)
309            return (track_pop, track_rate, track_ltime, album_mtime,
310                    track_loved, album_loved, album_pop, album_rate)
311        except Exception as e:
312            Logger.error("CollectionScanner::del_from_db: %s" % e)
313        return (0, 0, 0, 0, False, False, 0, 0)
314
315    def is_locked(self):
316        """
317            True if db locked
318            @return bool
319        """
320        return self.__thread is not None and self.__thread.is_alive()
321
322    def stop(self):
323        """
324            Stop scan
325        """
326        self.__thread = None
327
328    def reset_database(self):
329        """
330            Reset database
331        """
332        from lollypop.app_notification import AppNotification
333        App().window.container.progress.add(self)
334        App().window.container.progress.set_fraction(0, self)
335        self.__progress_fraction = 0
336        notification = AppNotification(_("Resetting database"), [], [], 10000)
337        notification.show()
338        App().window.container.add_overlay(notification)
339        notification.set_reveal_child(True)
340        App().task_helper.run(self.__reset_database)
341
342    @property
343    def inotify(self):
344        """
345            Get Inotify object
346            @return Inotify
347        """
348        return self.__inotify
349
350#######################
351# PRIVATE             #
352#######################
353    def __reset_database(self):
354        """
355            Reset database
356        """
357        def update_ui():
358            App().window.container.go_home()
359            App().scanner.update(ScanType.FULL)
360        App().player.stop()
361        if App().ws_director.collection_ws is not None:
362            App().ws_director.collection_ws.stop()
363        uris = App().tracks.get_uris()
364        i = 0
365        SqlCursor.add(App().db)
366        SqlCursor.add(self.__history)
367        count = len(uris)
368        for uri in uris:
369            self.del_from_db(uri, True)
370            self.__update_progress(i, count, 0.01)
371            i += 1
372        App().tracks.del_persistent(False)
373        App().tracks.clean(False)
374        App().albums.clean(False)
375        App().artists.clean(False)
376        App().genres.clean(False)
377        App().cache.clear_table("duration")
378        SqlCursor.commit(App().db)
379        SqlCursor.remove(App().db)
380        SqlCursor.commit(self.__history)
381        SqlCursor.remove(self.__history)
382        GLib.idle_add(update_ui)
383
384    def __update_progress(self, current, total, allowed_diff):
385        """
386            Update progress bar status
387            @param current as int
388            @param total as int
389            @param allowed_diff as float => allows to prevent
390                                            main loop flooding
391        """
392        new_fraction = current / total
393        if new_fraction > self.__progress_fraction + allowed_diff:
394            self.__progress_fraction = new_fraction
395            GLib.idle_add(App().window.container.progress.set_fraction,
396                          new_fraction, self)
397
398    def __finish(self, items):
399        """
400            Notify from main thread when scan finished
401            @param items as [CollectionItem]
402        """
403        track_ids = [item.track_id for item in items]
404        self.__thread = None
405        Logger.info("Scan finished")
406        App().lookup_action("update_db").set_enabled(True)
407        App().window.container.progress.set_fraction(1.0, self)
408        self.stop()
409        emit_signal(self, "scan-finished", track_ids)
410        # Update max count value
411        App().albums.update_max_count()
412        # Update featuring
413        App().artists.update_featuring()
414        if App().ws_director.collection_ws is not None:
415            App().ws_director.collection_ws.start()
416
417    def __add_monitor(self, dirs):
418        """
419            Monitor any change in a list of directory
420            @param dirs as str or list of directory to be monitored
421        """
422        if self.__inotify is None:
423            return
424        # Add monitors on dirs
425        for d in dirs:
426            # Handle a stop request
427            if self.__thread is None:
428                break
429            if d.startswith("file://"):
430                self.__inotify.add_monitor(d)
431
432    @profile
433    def __get_objects_for_uris(self, scan_type, uris):
434        """
435            Get all tracks and dirs in uris
436            @param scan_type as ScanType
437            @param uris as string
438            @return ([(int, str)], [str], [str])
439                    ([(mtime, file)], [dir], [stream])
440        """
441        files = []
442        dirs = []
443        streams = []
444        walk_uris = []
445        # Check collection exists
446        for uri in uris:
447            parsed = urlparse(uri)
448            if parsed.scheme in ["http", "https"]:
449                streams.append(uri)
450            else:
451                f = Gio.File.new_for_uri(uri)
452                if f.query_exists():
453                    walk_uris.append(uri)
454                else:
455                    return ([], [], [])
456
457        while walk_uris:
458            uri = walk_uris.pop(0)
459            try:
460                # Directly add files, walk through directories
461                f = Gio.File.new_for_uri(uri)
462                info = f.query_info(SCAN_QUERY_INFO,
463                                    Gio.FileQueryInfoFlags.NONE,
464                                    None)
465                if info.get_file_type() == Gio.FileType.DIRECTORY:
466                    dirs.append(uri)
467                    infos = f.enumerate_children(SCAN_QUERY_INFO,
468                                                 Gio.FileQueryInfoFlags.NONE,
469                                                 None)
470                    for info in infos:
471                        f = infos.get_child(info)
472                        child_uri = f.get_uri()
473                        if info.get_is_hidden():
474                            continue
475                        # User do not want internal symlinks
476                        elif info.get_is_symlink() and\
477                                App().settings.get_value("ignore-symlinks"):
478                            continue
479                        walk_uris.append(child_uri)
480                    infos.close(None)
481                # Only happens if files passed as args
482                else:
483                    mtime = get_mtime(info)
484                    files.append((mtime, uri))
485            except Exception as e:
486                Logger.error("CollectionScanner::__get_objects_for_uris(): %s"
487                             % e)
488        files.sort(reverse=True)
489        return (files, dirs, streams)
490
491    @profile
492    def __scan(self, scan_type, uris):
493        """
494            Scan music collection for music files
495            @param scan_type as ScanType
496            @param uris as [str]
497            @thread safe
498        """
499        try:
500            self.__items = []
501            App().art.clean_rounded()
502            (files, dirs, streams) = self.__get_objects_for_uris(
503                scan_type, uris)
504            if len(uris) != len(streams) and not files:
505                self.__flatpak_migration()
506                App().notify.send("Lollypop",
507                                  _("Scan disabled, missing collection"))
508                return
509            if scan_type == ScanType.NEW_FILES:
510                db_uris = App().tracks.get_uris(uris)
511            else:
512                db_uris = App().tracks.get_uris()
513
514            # Get mtime of all tracks to detect which has to be updated
515            db_mtimes = App().tracks.get_mtimes()
516            # * 2 => Scan + Save
517            self.__progress_total = len(files) * 2 + len(streams)
518            self.__progress_count = 0
519            self.__progress_fraction = 0
520            # Min: 1 thread, Max: 5 threads
521            count = max(1, min(5, cpu_count() // 2))
522            split_files = split_list(files, count)
523            self.__tags = {}
524            self.__notified_ids = []
525            self.__pending_new_artist_ids = []
526            threads = []
527            for files in split_files:
528                thread = App().task_helper.run(self.__scan_files,
529                                               files, db_mtimes,
530                                               scan_type)
531                threads.append(thread)
532            while threads:
533                sleep(0.1)
534                thread = threads[0]
535                if not thread.is_alive():
536                    threads.remove(thread)
537
538            SqlCursor.add(App().db)
539            if scan_type == ScanType.EXTERNAL:
540                storage_type = StorageType.EXTERNAL
541            else:
542                storage_type = StorageType.COLLECTION
543            self.__items += self.__save_in_db(storage_type)
544            # Add streams to DB, only happening on command line/m3u files
545            self.__items += self.__save_streams_in_db(streams, storage_type)
546
547            self.__remove_old_tracks(db_uris, scan_type)
548
549            if scan_type == ScanType.EXTERNAL:
550                albums = tracks_to_albums(
551                    [Track(item.track_id) for item in self.__items])
552                App().player.play_albums(albums)
553            else:
554                self.__add_monitor(dirs)
555                GLib.idle_add(self.__finish, self.__items)
556            self.__tags = {}
557            self.__items = []
558            self.__pending_new_artist_ids = []
559        except Exception as e:
560            Logger.warning("CollectionScanner::__scan(): %s", e)
561        SqlCursor.remove(App().db)
562        App().settings.set_value("flatpak-access-migration",
563                                 GLib.Variant("b", True))
564
565    def __scan_to_handle(self, uri):
566        """
567            Check if file has to be handle by scanner
568            @param f as Gio.File
569            @return bool
570        """
571        try:
572            file_type = get_file_type(uri)
573            # Get file type using Gio (slower)
574            if file_type == FileType.UNKNOWN:
575                f = Gio.File.new_for_uri(uri)
576                info = f.query_info(FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
577                                    Gio.FileQueryInfoFlags.NONE)
578                if is_pls(info):
579                    file_type = FileType.PLS
580                elif is_audio(info):
581                    file_type = FileType.AUDIO
582            if file_type == FileType.PLS:
583                Logger.debug("Importing playlist %s" % uri)
584                if App().settings.get_value("import-playlists"):
585                    App().playlists.import_tracks(uri)
586            elif file_type == FileType.AUDIO:
587                Logger.debug("Importing audio %s" % uri)
588                return True
589        except Exception as e:
590            Logger.error("CollectionScanner::__scan_to_handle(): %s" % e)
591        return False
592
593    def __scan_files(self, files, db_mtimes, scan_type):
594        """
595            Scan music collection for new audio files
596            @param files as [str]
597            @param db_mtimes as {}
598            @param scan_type as ScanType
599            @thread safe
600        """
601        discoverer = Discoverer()
602        try:
603            # Scan new files
604            for (mtime, uri) in files:
605                # Handle a stop request
606                if self.__thread is None and scan_type != ScanType.EXTERNAL:
607                    raise Exception("cancelled")
608                try:
609                    if not self.__scan_to_handle(uri):
610                        self.__progress_count += 2
611                        continue
612                    db_mtime = db_mtimes.get(uri, 0)
613                    if mtime > db_mtime:
614                        # Do not use mtime if not intial scan
615                        if db_mtimes:
616                            mtime = int(time())
617                        self.__tags[uri] = self.__get_tags(discoverer,
618                                                           uri, mtime)
619                        self.__progress_count += 1
620                        self.__update_progress(self.__progress_count,
621                                               self.__progress_total,
622                                               0.001)
623                    else:
624                        # We want to play files, so put them in items
625                        if scan_type == ScanType.EXTERNAL:
626                            track_id = App().tracks.get_id_by_uri(uri)
627                            item = CollectionItem(track_id=track_id)
628                            self.__items.append(item)
629                        self.__progress_count += 2
630                        self.__update_progress(self.__progress_count,
631                                               self.__progress_total,
632                                               0.1)
633                except Exception as e:
634                    Logger.error("Scanning file: %s, %s" % (uri, e))
635        except Exception as e:
636            Logger.warning("CollectionScanner::__scan_files(): % s" % e)
637
638    def __save_in_db(self, storage_type):
639        """
640            Save current tags into DB
641            @param storage_type as StorageType
642            @return [CollectionItem]
643        """
644        items = []
645        for uri in list(self.__tags.keys()):
646            # Handle a stop request
647            if self.__thread is None:
648                raise Exception("cancelled")
649            Logger.debug("Adding file: %s" % uri)
650            tags = self.__tags[uri]
651            item = self.__add2db(uri, *tags, storage_type)
652            items.append(item)
653            self.__progress_count += 1
654            self.__update_progress(self.__progress_count,
655                                   self.__progress_total,
656                                   0.001)
657            if item.album_id not in self.__notified_ids:
658                self.__notified_ids.append(item.album_id)
659                self.__notify_ui(item)
660            del self.__tags[uri]
661        # Handle a stop request
662        if self.__thread is None:
663            raise Exception("cancelled")
664        return items
665
666    def __save_streams_in_db(self, streams, storage_type):
667        """
668            Save http stream to DB
669            @param streams as [str]
670            @param storage_type as StorageType
671            @return [CollectionItem]
672        """
673        items = []
674        for uri in streams:
675            parsed = urlparse(uri)
676            item = self.__add2db(uri, parsed.path, parsed.netloc,
677                                 None, "", "", parsed.netloc,
678                                 parsed.netloc, "", False, 0, False, 0, 0, 0,
679                                 None, 0, "", "", "", "", 1, 0, 0, 0, 0, 0,
680                                 False, 0, False, storage_type)
681            items.append(item)
682            self.__progress_count += 1
683        return items
684
685    def __notify_ui(self, item):
686        """
687            Notify UI for item
688            @param items as CollectionItem
689        """
690        SqlCursor.commit(App().db)
691        if item.new_album:
692            emit_signal(self, "updated", item, ScanUpdate.ADDED)
693        else:
694            emit_signal(self, "updated", item, ScanUpdate.MODIFIED)
695
696    def __remove_old_tracks(self, uris, scan_type):
697        """
698            Remove non existent tracks from DB
699            @param scan_type as ScanType
700        """
701        if scan_type != ScanType.EXTERNAL and self.__thread is not None:
702            # We need to check files are always in collections
703            if scan_type == ScanType.FULL:
704                collections = App().settings.get_music_uris()
705            else:
706                collections = None
707            for uri in uris:
708                # Handle a stop request
709                if self.__thread is None:
710                    raise Exception("cancelled")
711                in_collection = True
712                if collections is not None:
713                    in_collection = False
714                    for collection in collections:
715                        if collection in uri:
716                            in_collection = True
717                            break
718                f = Gio.File.new_for_uri(uri)
719                if not in_collection:
720                    Logger.warning(
721                        "Removed, not in collection anymore: %s -> %s",
722                        uri, collections)
723                    self.del_from_db(uri, True)
724                elif not f.query_exists():
725                    Logger.warning("Removed, file has been deleted: %s", uri)
726                    self.del_from_db(uri, True)
727
728    def __get_tags(self, discoverer, uri, track_mtime):
729        """
730            Read track tags
731            @param discoverer as Discoverer
732            @param uri as string
733            @param track_mtime as int
734            @return ()
735        """
736        f = Gio.File.new_for_uri(uri)
737        info = discoverer.get_info(uri)
738        tags = info.get_tags()
739        name = f.get_basename()
740        duration = int(info.get_duration() / 1000000)
741        Logger.debug("CollectionScanner::add2db(): Restore stats")
742        # Restore stats
743        track_id = App().tracks.get_id_by_uri(uri)
744        if track_id is None:
745            track_id = App().tracks.get_id_by_basename_duration(name,
746                                                                duration)
747        if track_id is None:
748            (track_pop, track_rate, track_ltime,
749             album_mtime, track_loved, album_loved,
750             album_pop, album_rate, album_synced) = self.__history.get(
751                name, duration)
752        # Delete track and restore from it
753        else:
754            (track_pop, track_rate, track_ltime,
755             album_mtime, track_loved, album_loved,
756             album_pop, album_rate) = self.del_from_db(uri, False)
757
758        Logger.debug("CollectionScanner::add2db(): Read tags")
759        title = self.get_title(tags, name)
760        version = self.get_version(tags)
761        if version != "":
762            title += " (%s)" % version
763        artists = self.get_artists(tags)
764        a_sortnames = self.get_artist_sortnames(tags)
765        aa_sortnames = self.get_album_artist_sortnames(tags)
766        album_artists = self.get_album_artists(tags)
767        album_name = self.get_album_name(tags)
768        album_synced = 0
769        mb_album_id = self.get_mb_album_id(tags)
770        mb_track_id = self.get_mb_track_id(tags)
771        mb_artist_id = self.get_mb_artist_id(tags)
772        mb_album_artist_id = self.get_mb_album_artist_id(tags)
773        genres = self.get_genres(tags)
774        discnumber = self.get_discnumber(tags)
775        discname = self.get_discname(tags)
776        tracknumber = self.get_tracknumber(tags, name)
777        # We have popm in tags, override history one
778        tag_track_rate = self.get_popm(tags)
779        if tag_track_rate > 0:
780            track_rate = tag_track_rate
781        if album_mtime == 0:
782            album_mtime = track_mtime
783        bpm = self.get_bpm(tags)
784        compilation = self.get_compilation(tags)
785        (original_year, original_timestamp) = self.get_original_year(tags)
786        (year, timestamp) = self.get_year(tags)
787        if year is None:
788            (year, timestamp) = (original_year, original_timestamp)
789        elif original_year is None:
790            (original_year, original_timestamp) = (year, timestamp)
791        # If no artists tag, use album artist
792        if artists == "":
793            artists = album_artists
794        if App().settings.get_value("import-advanced-artist-tags"):
795            composers = self.get_composers(tags)
796            conductors = self.get_conductors(tags)
797            performers = self.get_performers(tags)
798            remixers = self.get_remixers(tags)
799            artists += ";%s" % performers if performers != "" else ""
800            artists += ";%s" % conductors if conductors != "" else ""
801            artists += ";%s" % composers if composers != "" else ""
802            artists += ";%s" % remixers if remixers != "" else ""
803        if artists == "":
804            artists = _("Unknown")
805        return (title, artists, genres, a_sortnames, aa_sortnames,
806                album_artists, album_name, discname, album_loved, album_mtime,
807                album_synced, album_rate, album_pop, discnumber, year,
808                timestamp, original_year, original_timestamp,
809                mb_album_id, mb_track_id, mb_artist_id,
810                mb_album_artist_id, tracknumber, track_pop, track_rate, bpm,
811                track_mtime, track_ltime, track_loved, duration, compilation)
812
813    def __add2db(self, uri, name, artists,
814                 genres, a_sortnames, aa_sortnames, album_artists, album_name,
815                 discname, album_loved, album_mtime, album_synced, album_rate,
816                 album_pop, discnumber, year, timestamp,
817                 original_year, original_timestamp, mb_album_id,
818                 mb_track_id, mb_artist_id, mb_album_artist_id,
819                 tracknumber, track_pop, track_rate, bpm, track_mtime,
820                 track_ltime, track_loved, duration, compilation,
821                 storage_type=StorageType.COLLECTION):
822        """
823            Add new file to DB
824            @param uri as str
825            @param tags as *()
826            @param storage_type as StorageType
827            @return CollectionItem
828        """
829        item = CollectionItem(uri=uri,
830                              track_name=name,
831                              artists=artists,
832                              genres=genres,
833                              a_sortnames=a_sortnames,
834                              aa_sortnames=aa_sortnames,
835                              album_artists=album_artists,
836                              album_name=album_name,
837                              discname=discname,
838                              album_loved=album_loved,
839                              album_mtime=album_mtime,
840                              album_synced=album_synced,
841                              album_rate=album_rate,
842                              album_pop=album_pop,
843                              discnumber=discnumber,
844                              year=year,
845                              timestamp=timestamp,
846                              original_year=original_year,
847                              original_timestamp=original_timestamp,
848                              mb_album_id=mb_album_id,
849                              mb_track_id=mb_track_id,
850                              mb_artist_id=mb_artist_id,
851                              mb_album_artist_id=mb_album_artist_id,
852                              tracknumber=tracknumber,
853                              track_pop=track_pop,
854                              track_rate=track_rate,
855                              bpm=bpm,
856                              track_mtime=track_mtime,
857                              track_ltime=track_ltime,
858                              track_loved=track_loved,
859                              duration=duration,
860                              compilation=compilation,
861                              storage_type=storage_type)
862        self.save_album(item)
863        self.save_track(item)
864        return item
865
866    def __flatpak_migration(self):
867        """
868            https://github.com/flathub/org.gnome.Lollypop/pull/108
869        """
870        if GLib.file_test("/app", GLib.FileTest.EXISTS) and\
871                not App().settings.get_value("flatpak-access-migration"):
872            from lollypop.assistant_flatpak import FlatpakAssistant
873            assistant = FlatpakAssistant()
874            assistant.set_position(Gtk.WindowPosition.CENTER_ON_PARENT)
875            assistant.set_transient_for(App().window)
876            GLib.timeout_add(1000, assistant.show)
877