1#!/usr/bin/python3 -OO
2# Copyright 2007-2021 The SABnzbd-Team <team@sabnzbd.org>
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License
6# as published by the Free Software Foundation; either version 2
7# of the License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
17
18"""
19sabnzbd.dirscanner - Scanner for Watched Folder
20"""
21
22import os
23import time
24import logging
25import threading
26
27import sabnzbd
28from sabnzbd.constants import SCAN_FILE_NAME, VALID_ARCHIVES, VALID_NZB_FILES
29import sabnzbd.filesystem as filesystem
30import sabnzbd.config as config
31import sabnzbd.cfg as cfg
32
33
34def compare_stat_tuple(tup1, tup2):
35    """Test equality of two stat-tuples, content-related parts only"""
36    if tup1.st_ino != tup2.st_ino:
37        return False
38    if tup1.st_size != tup2.st_size:
39        return False
40    if tup1.st_mtime != tup2.st_mtime:
41        return False
42    if tup1.st_ctime != tup2.st_ctime:
43        return False
44    return True
45
46
47def clean_file_list(inp_list, folder, files):
48    """Remove elements of "inp_list" not found in "files" """
49    for path in sorted(inp_list):
50        fld, name = os.path.split(path)
51        if fld == folder:
52            present = False
53            for name in files:
54                if os.path.join(folder, name) == path:
55                    present = True
56                    break
57            if not present:
58                del inp_list[path]
59
60
61class DirScanner(threading.Thread):
62    """Thread that periodically scans a given directory and picks up any
63    valid NZB, NZB.GZ ZIP-with-only-NZB and even NZB.GZ named as .NZB
64    Candidates which turned out wrong, will be remembered and skipped in
65    subsequent scans, unless changed.
66    """
67
68    def __init__(self):
69        super().__init__()
70
71        self.newdir()
72        try:
73            dirscan_dir, self.ignored, self.suspected = sabnzbd.load_admin(SCAN_FILE_NAME)
74            if dirscan_dir != self.dirscan_dir:
75                self.ignored = {}
76                self.suspected = {}
77        except:
78            self.ignored = {}  # Will hold all unusable files and the
79            # successfully processed ones that cannot be deleted
80            self.suspected = {}  # Will hold name/attributes of suspected candidates
81
82        self.loop_condition = threading.Condition(threading.Lock())
83        self.shutdown = False
84        self.error_reported = False  # Prevents multiple reporting of missing watched folder
85        self.dirscan_dir = cfg.dirscan_dir.get_path()
86        self.dirscan_speed = cfg.dirscan_speed() or None  # If set to 0, use None so the wait() is forever
87        self.busy = False
88        cfg.dirscan_dir.callback(self.newdir)
89        cfg.dirscan_speed.callback(self.newspeed)
90
91    def newdir(self):
92        """We're notified of a dir change"""
93        self.ignored = {}
94        self.suspected = {}
95        self.dirscan_dir = cfg.dirscan_dir.get_path()
96        self.dirscan_speed = cfg.dirscan_speed()
97
98    def newspeed(self):
99        """We're notified of a scan speed change"""
100        # If set to 0, use None so the wait() is forever
101        self.dirscan_speed = cfg.dirscan_speed() or None
102        with self.loop_condition:
103            self.loop_condition.notify()
104
105    def stop(self):
106        """Stop the dir scanner"""
107        self.shutdown = True
108        with self.loop_condition:
109            self.loop_condition.notify()
110
111    def save(self):
112        """Save dir scanner bookkeeping"""
113        sabnzbd.save_admin((self.dirscan_dir, self.ignored, self.suspected), SCAN_FILE_NAME)
114
115    def run(self):
116        """Start the scanner"""
117        logging.info("Dirscanner starting up")
118        self.shutdown = False
119
120        while not self.shutdown:
121            # Wait to be woken up or triggered
122            with self.loop_condition:
123                self.loop_condition.wait(self.dirscan_speed)
124            if self.dirscan_speed and not self.shutdown:
125                self.scan()
126
127    def scan(self):
128        """Do one scan of the watched folder"""
129
130        def run_dir(folder, catdir):
131            try:
132                files = os.listdir(folder)
133            except OSError:
134                if not self.error_reported and not catdir:
135                    logging.error(T("Cannot read Watched Folder %s"), filesystem.clip_path(folder))
136                    self.error_reported = True
137                files = []
138
139            for filename in files:
140                if self.shutdown:
141                    break
142                path = os.path.join(folder, filename)
143                if os.path.isdir(path) or path in self.ignored or filename[0] == ".":
144                    continue
145
146                if filesystem.get_ext(path) in VALID_NZB_FILES + VALID_ARCHIVES:
147                    try:
148                        stat_tuple = os.stat(path)
149                    except OSError:
150                        continue
151                else:
152                    self.ignored[path] = 1
153                    continue
154
155                if path in self.suspected:
156                    if compare_stat_tuple(self.suspected[path], stat_tuple):
157                        # Suspected file still has the same attributes
158                        continue
159                    else:
160                        del self.suspected[path]
161
162                if stat_tuple.st_size > 0:
163                    logging.info("Trying to import %s", path)
164
165                    # Wait until the attributes are stable for 1 second, but give up after 3 sec
166                    # This indicates that the file is fully written to disk
167                    for n in range(3):
168                        time.sleep(1.0)
169                        try:
170                            stat_tuple_tmp = os.stat(path)
171                        except OSError:
172                            continue
173                        if compare_stat_tuple(stat_tuple, stat_tuple_tmp):
174                            break
175                        stat_tuple = stat_tuple_tmp
176                    else:
177                        # Not stable
178                        continue
179
180                    # Add the NZB's
181                    res, _ = sabnzbd.add_nzbfile(path, catdir=catdir, keep=False)
182                    if res < 0:
183                        # Retry later, for example when we can't read the file
184                        self.suspected[path] = stat_tuple
185                    elif res == 0:
186                        self.error_reported = False
187                    else:
188                        self.ignored[path] = 1
189
190            # Remove files from the bookkeeping that are no longer on the disk
191            clean_file_list(self.ignored, folder, files)
192            clean_file_list(self.suspected, folder, files)
193
194        if not self.busy:
195            self.busy = True
196            dirscan_dir = self.dirscan_dir
197            if dirscan_dir and not sabnzbd.PAUSED_ALL:
198                run_dir(dirscan_dir, None)
199
200                try:
201                    dirscan_list = os.listdir(dirscan_dir)
202                except OSError:
203                    if not self.error_reported:
204                        logging.error(T("Cannot read Watched Folder %s"), filesystem.clip_path(dirscan_dir))
205                        self.error_reported = True
206                    dirscan_list = []
207
208                cats = config.get_categories()
209                for dd in dirscan_list:
210                    dpath = os.path.join(dirscan_dir, dd)
211                    if os.path.isdir(dpath) and dd.lower() in cats:
212                        run_dir(dpath, dd.lower())
213            self.busy = False
214