1#!/usr/bin/python3 -OO 2# Copyright 2007-2021 The SABnzbd-Team <team@sabnzbd.org> 3# 4# This program is free software; you can redistribute it and/or 5# modify it under the terms of the GNU General Public License 6# as published by the Free Software Foundation; either version 2 7# of the License, or (at your option) any later version. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License 15# along with this program; if not, write to the Free Software 16# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 18""" 19sabnzbd.dirscanner - Scanner for Watched Folder 20""" 21 22import os 23import time 24import logging 25import threading 26 27import sabnzbd 28from sabnzbd.constants import SCAN_FILE_NAME, VALID_ARCHIVES, VALID_NZB_FILES 29import sabnzbd.filesystem as filesystem 30import sabnzbd.config as config 31import sabnzbd.cfg as cfg 32 33 34def compare_stat_tuple(tup1, tup2): 35 """Test equality of two stat-tuples, content-related parts only""" 36 if tup1.st_ino != tup2.st_ino: 37 return False 38 if tup1.st_size != tup2.st_size: 39 return False 40 if tup1.st_mtime != tup2.st_mtime: 41 return False 42 if tup1.st_ctime != tup2.st_ctime: 43 return False 44 return True 45 46 47def clean_file_list(inp_list, folder, files): 48 """Remove elements of "inp_list" not found in "files" """ 49 for path in sorted(inp_list): 50 fld, name = os.path.split(path) 51 if fld == folder: 52 present = False 53 for name in files: 54 if os.path.join(folder, name) == path: 55 present = True 56 break 57 if not present: 58 del inp_list[path] 59 60 61class DirScanner(threading.Thread): 62 """Thread that periodically scans a given directory and picks up any 63 valid NZB, NZB.GZ ZIP-with-only-NZB and even NZB.GZ named as .NZB 64 Candidates which turned out wrong, will be remembered and skipped in 65 subsequent scans, unless changed. 66 """ 67 68 def __init__(self): 69 super().__init__() 70 71 self.newdir() 72 try: 73 dirscan_dir, self.ignored, self.suspected = sabnzbd.load_admin(SCAN_FILE_NAME) 74 if dirscan_dir != self.dirscan_dir: 75 self.ignored = {} 76 self.suspected = {} 77 except: 78 self.ignored = {} # Will hold all unusable files and the 79 # successfully processed ones that cannot be deleted 80 self.suspected = {} # Will hold name/attributes of suspected candidates 81 82 self.loop_condition = threading.Condition(threading.Lock()) 83 self.shutdown = False 84 self.error_reported = False # Prevents multiple reporting of missing watched folder 85 self.dirscan_dir = cfg.dirscan_dir.get_path() 86 self.dirscan_speed = cfg.dirscan_speed() or None # If set to 0, use None so the wait() is forever 87 self.busy = False 88 cfg.dirscan_dir.callback(self.newdir) 89 cfg.dirscan_speed.callback(self.newspeed) 90 91 def newdir(self): 92 """We're notified of a dir change""" 93 self.ignored = {} 94 self.suspected = {} 95 self.dirscan_dir = cfg.dirscan_dir.get_path() 96 self.dirscan_speed = cfg.dirscan_speed() 97 98 def newspeed(self): 99 """We're notified of a scan speed change""" 100 # If set to 0, use None so the wait() is forever 101 self.dirscan_speed = cfg.dirscan_speed() or None 102 with self.loop_condition: 103 self.loop_condition.notify() 104 105 def stop(self): 106 """Stop the dir scanner""" 107 self.shutdown = True 108 with self.loop_condition: 109 self.loop_condition.notify() 110 111 def save(self): 112 """Save dir scanner bookkeeping""" 113 sabnzbd.save_admin((self.dirscan_dir, self.ignored, self.suspected), SCAN_FILE_NAME) 114 115 def run(self): 116 """Start the scanner""" 117 logging.info("Dirscanner starting up") 118 self.shutdown = False 119 120 while not self.shutdown: 121 # Wait to be woken up or triggered 122 with self.loop_condition: 123 self.loop_condition.wait(self.dirscan_speed) 124 if self.dirscan_speed and not self.shutdown: 125 self.scan() 126 127 def scan(self): 128 """Do one scan of the watched folder""" 129 130 def run_dir(folder, catdir): 131 try: 132 files = os.listdir(folder) 133 except OSError: 134 if not self.error_reported and not catdir: 135 logging.error(T("Cannot read Watched Folder %s"), filesystem.clip_path(folder)) 136 self.error_reported = True 137 files = [] 138 139 for filename in files: 140 if self.shutdown: 141 break 142 path = os.path.join(folder, filename) 143 if os.path.isdir(path) or path in self.ignored or filename[0] == ".": 144 continue 145 146 if filesystem.get_ext(path) in VALID_NZB_FILES + VALID_ARCHIVES: 147 try: 148 stat_tuple = os.stat(path) 149 except OSError: 150 continue 151 else: 152 self.ignored[path] = 1 153 continue 154 155 if path in self.suspected: 156 if compare_stat_tuple(self.suspected[path], stat_tuple): 157 # Suspected file still has the same attributes 158 continue 159 else: 160 del self.suspected[path] 161 162 if stat_tuple.st_size > 0: 163 logging.info("Trying to import %s", path) 164 165 # Wait until the attributes are stable for 1 second, but give up after 3 sec 166 # This indicates that the file is fully written to disk 167 for n in range(3): 168 time.sleep(1.0) 169 try: 170 stat_tuple_tmp = os.stat(path) 171 except OSError: 172 continue 173 if compare_stat_tuple(stat_tuple, stat_tuple_tmp): 174 break 175 stat_tuple = stat_tuple_tmp 176 else: 177 # Not stable 178 continue 179 180 # Add the NZB's 181 res, _ = sabnzbd.add_nzbfile(path, catdir=catdir, keep=False) 182 if res < 0: 183 # Retry later, for example when we can't read the file 184 self.suspected[path] = stat_tuple 185 elif res == 0: 186 self.error_reported = False 187 else: 188 self.ignored[path] = 1 189 190 # Remove files from the bookkeeping that are no longer on the disk 191 clean_file_list(self.ignored, folder, files) 192 clean_file_list(self.suspected, folder, files) 193 194 if not self.busy: 195 self.busy = True 196 dirscan_dir = self.dirscan_dir 197 if dirscan_dir and not sabnzbd.PAUSED_ALL: 198 run_dir(dirscan_dir, None) 199 200 try: 201 dirscan_list = os.listdir(dirscan_dir) 202 except OSError: 203 if not self.error_reported: 204 logging.error(T("Cannot read Watched Folder %s"), filesystem.clip_path(dirscan_dir)) 205 self.error_reported = True 206 dirscan_list = [] 207 208 cats = config.get_categories() 209 for dd in dirscan_list: 210 dpath = os.path.join(dirscan_dir, dd) 211 if os.path.isdir(dpath) and dd.lower() in cats: 212 run_dir(dpath, dd.lower()) 213 self.busy = False 214