1import re 2import sys 3from datetime import datetime 4from pathlib import Path 5 6from loguru import logger 7 8from flexget import plugin 9from flexget.config_schema import one_or_more 10from flexget.entry import Entry 11from flexget.event import event 12 13logger = logger.bind(name='filesystem') 14 15 16class Filesystem: 17 """ 18 Uses local path content as an input. Can use recursion if configured. 19 Recursion is False by default. Can be configured to true or get integer that will specify max depth in relation to 20 base folder. 21 All files/dir/symlinks are retrieved by default. Can be changed by using the 'retrieve' property. 22 23 Example 1:: Single path 24 25 filesystem: /storage/movies/ 26 27 Example 2:: List of paths 28 29 filesystem: 30 - /storage/movies/ 31 - /storage/tv/ 32 33 Example 3:: Object with list of paths 34 35 filesystem: 36 path: 37 - /storage/movies/ 38 - /storage/tv/ 39 mask: '*.mkv' 40 41 Example 4:: 42 43 filesystem: 44 path: 45 - /storage/movies/ 46 - /storage/tv/ 47 recursive: 4 # 4 levels deep from each base folder 48 retrieve: files # Only files will be retrieved 49 50 Example 5:: 51 52 filesystem: 53 path: 54 - /storage/movies/ 55 - /storage/tv/ 56 recursive: yes # No limit to depth, all sub dirs will be accessed 57 retrieve: # Only files and dirs will be retrieved 58 - files 59 - dirs 60 61 """ 62 63 retrieval_options = ['files', 'dirs', 'symlinks'] 64 paths = one_or_more({'type': 'string', 'format': 'path'}, unique_items=True) 65 66 schema = { 67 'oneOf': [ 68 paths, 69 { 70 'type': 'object', 71 'properties': { 72 'path': paths, 73 'mask': {'type': 'string'}, 74 'regexp': {'type': 'string', 'format': 'regex'}, 75 'recursive': { 76 'oneOf': [{'type': 'integer', 'minimum': 2}, {'type': 'boolean'}] 77 }, 78 'retrieve': one_or_more( 79 {'type': 'string', 'enum': retrieval_options}, unique_items=True 80 ), 81 }, 82 'required': ['path'], 83 'additionalProperties': False, 84 }, 85 ] 86 } 87 88 def prepare_config(self, config): 89 from fnmatch import translate 90 91 config = config 92 93 # Converts config to a dict with a list of paths 94 if not isinstance(config, dict): 95 config = {'path': config} 96 if not isinstance(config['path'], list): 97 config['path'] = [config['path']] 98 99 config.setdefault('recursive', False) 100 # If mask was specified, turn it in to a regexp 101 if config.get('mask'): 102 config['regexp'] = translate(config['mask']) 103 # If no mask or regexp specified, accept all files 104 config.setdefault('regexp', '.') 105 # Sets the default retrieval option to files 106 config.setdefault('retrieve', self.retrieval_options) 107 108 return config 109 110 def create_entry(self, filepath: Path, test_mode): 111 """ 112 Creates a single entry using a filepath and a type (file/dir) 113 """ 114 filepath = filepath.absolute() 115 entry = Entry() 116 entry['location'] = str(filepath) 117 entry['url'] = Path(filepath).absolute().as_uri() 118 entry['filename'] = filepath.name 119 if filepath.is_file(): 120 entry['title'] = filepath.stem 121 else: 122 entry['title'] = filepath.name 123 file_stat = filepath.stat() 124 try: 125 entry['timestamp'] = datetime.fromtimestamp(file_stat.st_mtime) 126 except Exception as e: 127 logger.warning('Error setting timestamp for {}: {}', filepath, e) 128 entry['timestamp'] = None 129 entry['accessed'] = datetime.fromtimestamp(file_stat.st_atime) 130 entry['modified'] = datetime.fromtimestamp(file_stat.st_mtime) 131 entry['created'] = datetime.fromtimestamp(file_stat.st_ctime) 132 if entry.isvalid(): 133 if test_mode: 134 logger.info("Test mode. Entry includes:") 135 logger.info(' Title: {}', entry['title']) 136 logger.info(' URL: {}', entry['url']) 137 logger.info(' Filename: {}', entry['filename']) 138 logger.info(' Location: {}', entry['location']) 139 logger.info(' Timestamp: {}', entry['timestamp']) 140 return entry 141 else: 142 logger.error('Non valid entry created: {} ', entry) 143 return 144 145 def get_max_depth(self, recursion, base_depth): 146 if recursion is False: 147 return base_depth + 1 148 elif recursion is True: 149 return float('inf') 150 else: 151 return base_depth + recursion 152 153 @staticmethod 154 def get_folder_objects(folder: Path, recursion: bool): 155 return folder.rglob('*') if recursion else folder.iterdir() 156 157 def get_entries_from_path( 158 self, path_list, match, recursion, test_mode, get_files, get_dirs, get_symlinks 159 ): 160 entries = [] 161 162 for folder in path_list: 163 logger.verbose('Scanning folder {}. Recursion is set to {}.', folder, recursion) 164 folder = Path(folder).expanduser() 165 if not folder.exists(): 166 logger.error('{} does not exist (anymore.)', folder) 167 continue 168 logger.debug('Scanning {}', folder) 169 base_depth = len(folder.parts) 170 max_depth = self.get_max_depth(recursion, base_depth) 171 folder_objects = self.get_folder_objects(folder, recursion) 172 for path_object in folder_objects: 173 logger.debug('Checking if {} qualifies to be added as an entry.', path_object) 174 try: 175 path_object.exists() 176 except UnicodeError: 177 logger.error( 178 'File {} not decodable with filesystem encoding: {}', 179 path_object, 180 sys.getfilesystemencoding(), 181 ) 182 continue 183 entry = None 184 object_depth = len(path_object.parts) 185 if object_depth <= max_depth: 186 if match(str(path_object)): 187 if ( 188 (path_object.is_dir() and get_dirs) 189 or (path_object.is_symlink() and get_symlinks) 190 or ( 191 path_object.is_file() 192 and not path_object.is_symlink() 193 and get_files 194 ) 195 ): 196 entry = self.create_entry(path_object, test_mode) 197 else: 198 logger.debug( 199 "Path object's {} type doesn't match requested object types.", 200 path_object, 201 ) 202 if entry and entry not in entries: 203 entries.append(entry) 204 205 return entries 206 207 def on_task_input(self, task, config): 208 config = self.prepare_config(config) 209 210 path_list = config['path'] 211 test_mode = task.options.test 212 match = re.compile(config['regexp'], re.IGNORECASE).match 213 recursive = config['recursive'] 214 get_files = 'files' in config['retrieve'] 215 get_dirs = 'dirs' in config['retrieve'] 216 get_symlinks = 'symlinks' in config['retrieve'] 217 218 logger.verbose('Starting to scan folders.') 219 return self.get_entries_from_path( 220 path_list, match, recursive, test_mode, get_files, get_dirs, get_symlinks 221 ) 222 223 224@event('plugin.register') 225def register_plugin(): 226 plugin.register(Filesystem, 'filesystem', api_ver=2) 227