1import re
2import sys
3from datetime import datetime
4from pathlib import Path
5
6from loguru import logger
7
8from flexget import plugin
9from flexget.config_schema import one_or_more
10from flexget.entry import Entry
11from flexget.event import event
12
13logger = logger.bind(name='filesystem')
14
15
16class Filesystem:
17    """
18    Uses local path content as an input. Can use recursion if configured.
19    Recursion is False by default. Can be configured to true or get integer that will specify max depth in relation to
20        base folder.
21    All files/dir/symlinks are retrieved by default. Can be changed by using the 'retrieve' property.
22
23    Example 1:: Single path
24
25      filesystem: /storage/movies/
26
27    Example 2:: List of paths
28
29      filesystem:
30         - /storage/movies/
31         - /storage/tv/
32
33    Example 3:: Object with list of paths
34
35      filesystem:
36        path:
37          - /storage/movies/
38          - /storage/tv/
39        mask: '*.mkv'
40
41    Example 4::
42
43      filesystem:
44        path:
45          - /storage/movies/
46          - /storage/tv/
47        recursive: 4  # 4 levels deep from each base folder
48        retrieve: files  # Only files will be retrieved
49
50    Example 5::
51
52      filesystem:
53        path:
54          - /storage/movies/
55          - /storage/tv/
56        recursive: yes  # No limit to depth, all sub dirs will be accessed
57        retrieve:  # Only files and dirs will be retrieved
58          - files
59          - dirs
60
61    """
62
63    retrieval_options = ['files', 'dirs', 'symlinks']
64    paths = one_or_more({'type': 'string', 'format': 'path'}, unique_items=True)
65
66    schema = {
67        'oneOf': [
68            paths,
69            {
70                'type': 'object',
71                'properties': {
72                    'path': paths,
73                    'mask': {'type': 'string'},
74                    'regexp': {'type': 'string', 'format': 'regex'},
75                    'recursive': {
76                        'oneOf': [{'type': 'integer', 'minimum': 2}, {'type': 'boolean'}]
77                    },
78                    'retrieve': one_or_more(
79                        {'type': 'string', 'enum': retrieval_options}, unique_items=True
80                    ),
81                },
82                'required': ['path'],
83                'additionalProperties': False,
84            },
85        ]
86    }
87
88    def prepare_config(self, config):
89        from fnmatch import translate
90
91        config = config
92
93        # Converts config to a dict with a list of paths
94        if not isinstance(config, dict):
95            config = {'path': config}
96        if not isinstance(config['path'], list):
97            config['path'] = [config['path']]
98
99        config.setdefault('recursive', False)
100        # If mask was specified, turn it in to a regexp
101        if config.get('mask'):
102            config['regexp'] = translate(config['mask'])
103        # If no mask or regexp specified, accept all files
104        config.setdefault('regexp', '.')
105        # Sets the default retrieval option to files
106        config.setdefault('retrieve', self.retrieval_options)
107
108        return config
109
110    def create_entry(self, filepath: Path, test_mode):
111        """
112        Creates a single entry using a filepath and a type (file/dir)
113        """
114        filepath = filepath.absolute()
115        entry = Entry()
116        entry['location'] = str(filepath)
117        entry['url'] = Path(filepath).absolute().as_uri()
118        entry['filename'] = filepath.name
119        if filepath.is_file():
120            entry['title'] = filepath.stem
121        else:
122            entry['title'] = filepath.name
123        file_stat = filepath.stat()
124        try:
125            entry['timestamp'] = datetime.fromtimestamp(file_stat.st_mtime)
126        except Exception as e:
127            logger.warning('Error setting timestamp for {}: {}', filepath, e)
128            entry['timestamp'] = None
129        entry['accessed'] = datetime.fromtimestamp(file_stat.st_atime)
130        entry['modified'] = datetime.fromtimestamp(file_stat.st_mtime)
131        entry['created'] = datetime.fromtimestamp(file_stat.st_ctime)
132        if entry.isvalid():
133            if test_mode:
134                logger.info("Test mode. Entry includes:")
135                logger.info(' Title: {}', entry['title'])
136                logger.info(' URL: {}', entry['url'])
137                logger.info(' Filename: {}', entry['filename'])
138                logger.info(' Location: {}', entry['location'])
139                logger.info(' Timestamp: {}', entry['timestamp'])
140            return entry
141        else:
142            logger.error('Non valid entry created: {} ', entry)
143            return
144
145    def get_max_depth(self, recursion, base_depth):
146        if recursion is False:
147            return base_depth + 1
148        elif recursion is True:
149            return float('inf')
150        else:
151            return base_depth + recursion
152
153    @staticmethod
154    def get_folder_objects(folder: Path, recursion: bool):
155        return folder.rglob('*') if recursion else folder.iterdir()
156
157    def get_entries_from_path(
158        self, path_list, match, recursion, test_mode, get_files, get_dirs, get_symlinks
159    ):
160        entries = []
161
162        for folder in path_list:
163            logger.verbose('Scanning folder {}. Recursion is set to {}.', folder, recursion)
164            folder = Path(folder).expanduser()
165            if not folder.exists():
166                logger.error('{} does not exist (anymore.)', folder)
167                continue
168            logger.debug('Scanning {}', folder)
169            base_depth = len(folder.parts)
170            max_depth = self.get_max_depth(recursion, base_depth)
171            folder_objects = self.get_folder_objects(folder, recursion)
172            for path_object in folder_objects:
173                logger.debug('Checking if {} qualifies to be added as an entry.', path_object)
174                try:
175                    path_object.exists()
176                except UnicodeError:
177                    logger.error(
178                        'File {} not decodable with filesystem encoding: {}',
179                        path_object,
180                        sys.getfilesystemencoding(),
181                    )
182                    continue
183                entry = None
184                object_depth = len(path_object.parts)
185                if object_depth <= max_depth:
186                    if match(str(path_object)):
187                        if (
188                            (path_object.is_dir() and get_dirs)
189                            or (path_object.is_symlink() and get_symlinks)
190                            or (
191                                path_object.is_file()
192                                and not path_object.is_symlink()
193                                and get_files
194                            )
195                        ):
196                            entry = self.create_entry(path_object, test_mode)
197                        else:
198                            logger.debug(
199                                "Path object's {} type doesn't match requested object types.",
200                                path_object,
201                            )
202                        if entry and entry not in entries:
203                            entries.append(entry)
204
205        return entries
206
207    def on_task_input(self, task, config):
208        config = self.prepare_config(config)
209
210        path_list = config['path']
211        test_mode = task.options.test
212        match = re.compile(config['regexp'], re.IGNORECASE).match
213        recursive = config['recursive']
214        get_files = 'files' in config['retrieve']
215        get_dirs = 'dirs' in config['retrieve']
216        get_symlinks = 'symlinks' in config['retrieve']
217
218        logger.verbose('Starting to scan folders.')
219        return self.get_entries_from_path(
220            path_list, match, recursive, test_mode, get_files, get_dirs, get_symlinks
221        )
222
223
224@event('plugin.register')
225def register_plugin():
226    plugin.register(Filesystem, 'filesystem', api_ver=2)
227