1# -*- coding: utf-8 -*-
2#
3# Copyright (C) 2008 Andrew Resch <andrewresch@gmail.com>
4# Copyright (C) 2009-2010 John Garland <johnnybg+deluge@gmail.com>
5#
6# This file is part of Deluge and is licensed under GNU General Public License 3.0, or later, with
7# the additional special exception to link portions of this program with the OpenSSL library.
8# See LICENSE for more details.
9#
10
11from __future__ import division, unicode_literals
12
13import logging
14import os
15import shutil
16import time
17from datetime import datetime, timedelta
18from email.utils import formatdate
19
20from twisted.internet import defer, threads
21from twisted.internet.task import LoopingCall
22from twisted.web import error
23
24import deluge.component as component
25import deluge.configmanager
26from deluge.common import is_url
27from deluge.core.rpcserver import export
28from deluge.httpdownloader import download_file
29from deluge.plugins.pluginbase import CorePluginBase
30
31from .common import IP, BadIP
32from .detect import UnknownFormatError, create_reader, detect_compression, detect_format
33from .readers import ReaderParseError
34
35try:
36    from urllib.parse import urljoin
37except ImportError:
38    # PY2 fallback
39    from urlparse import urljoin  # pylint: disable=ungrouped-imports
40
41# TODO: check return values for deferred callbacks
42# TODO: review class attributes for redundancy
43
44log = logging.getLogger(__name__)
45
46DEFAULT_PREFS = {
47    'url': '',
48    'load_on_start': False,
49    'check_after_days': 4,
50    'list_compression': '',
51    'list_type': '',
52    'last_update': 0.0,
53    'list_size': 0,
54    'timeout': 180,
55    'try_times': 3,
56    'whitelisted': [],
57}
58
59# Constants
60ALLOW_RANGE = 0
61BLOCK_RANGE = 1
62
63
64class Core(CorePluginBase):
65    def enable(self):
66        log.debug('Blocklist: Plugin enabled...')
67
68        self.is_url = True
69        self.is_downloading = False
70        self.is_importing = False
71        self.has_imported = False
72        self.up_to_date = False
73        self.need_to_resume_session = False
74        self.num_whited = 0
75        self.num_blocked = 0
76        self.file_progress = 0.0
77
78        self.core = component.get('Core')
79        self.config = deluge.configmanager.ConfigManager(
80            'blocklist.conf', DEFAULT_PREFS
81        )
82        if 'whitelisted' not in self.config:
83            self.config['whitelisted'] = []
84
85        self.reader = create_reader(
86            self.config['list_type'], self.config['list_compression']
87        )
88
89        if not isinstance(self.config['last_update'], float):
90            self.config.config['last_update'] = 0.0
91
92        update_now = False
93        if self.config['load_on_start']:
94            self.pause_session()
95            if self.config['last_update']:
96                last_update = datetime.fromtimestamp(self.config['last_update'])
97                check_period = timedelta(days=self.config['check_after_days'])
98            if (
99                not self.config['last_update']
100                or last_update + check_period < datetime.now()
101            ):
102                update_now = True
103            else:
104                d = self.import_list(
105                    deluge.configmanager.get_config_dir('blocklist.cache')
106                )
107                d.addCallbacks(self.on_import_complete, self.on_import_error)
108                if self.need_to_resume_session:
109                    d.addBoth(self.resume_session)
110
111        # This function is called every 'check_after_days' days, to download
112        # and import a new list if needed.
113        self.update_timer = LoopingCall(self.check_import)
114        if self.config['check_after_days'] > 0:
115            self.update_timer.start(
116                self.config['check_after_days'] * 24 * 60 * 60, update_now
117            )
118
119    def disable(self):
120        self.config.save()
121        log.debug('Reset IP filter')
122        self.core.session.get_ip_filter().add_rule(
123            '0.0.0.0', '255.255.255.255', ALLOW_RANGE
124        )
125        log.debug('Blocklist: Plugin disabled')
126
127    def update(self):
128        pass
129
130    # Exported RPC methods #
131    @export
132    def check_import(self, force=False):
133        """Imports latest blocklist specified by blocklist url.
134
135        Args:
136            force (bool, optional): Force the download/import, default is False.
137
138        Returns:
139            Deferred: A Deferred which fires when the blocklist has been imported.
140
141        """
142        if not self.config['url']:
143            return
144
145        # Reset variables
146        self.filename = None
147        self.force_download = force
148        self.failed_attempts = 0
149        self.auto_detected = False
150        self.up_to_date = False
151        if force:
152            self.reader = None
153        self.is_url = is_url(self.config['url'])
154
155        # Start callback chain
156        if self.is_url:
157            d = self.download_list()
158            d.addCallbacks(self.on_download_complete, self.on_download_error)
159            d.addCallback(self.import_list)
160        else:
161            d = self.import_list(self.config['url'])
162        d.addCallbacks(self.on_import_complete, self.on_import_error)
163        if self.need_to_resume_session:
164            d.addBoth(self.resume_session)
165
166        return d
167
168    @export
169    def get_config(self):
170        """Gets the blocklist config dictionary.
171
172        Returns:
173            dict: The config dictionary.
174
175        """
176        return self.config.config
177
178    @export
179    def set_config(self, config):
180        """Sets the blocklist config.
181
182        Args:
183            config (dict): config to set.
184
185        """
186        needs_blocklist_import = False
187        for key in config:
188            if key == 'whitelisted':
189                saved = set(self.config[key])
190                update = set(config[key])
191                diff = saved.symmetric_difference(update)
192                if diff:
193                    log.debug('Whitelist changed. Updating...')
194                    added = update.intersection(diff)
195                    removed = saved.intersection(diff)
196                    if added:
197                        for ip in added:
198                            try:
199                                ip = IP.parse(ip)
200                                self.blocklist.add_rule(
201                                    ip.address, ip.address, ALLOW_RANGE
202                                )
203                                saved.add(ip.address)
204                                log.debug('Added %s to whitelisted', ip)
205                                self.num_whited += 1
206                            except BadIP as ex:
207                                log.error('Bad IP: %s', ex)
208                                continue
209                    if removed:
210                        needs_blocklist_import = True
211                        for ip in removed:
212                            try:
213                                ip = IP.parse(ip)
214                                saved.remove(ip.address)
215                                log.debug('Removed %s from whitelisted', ip)
216                            except BadIP as ex:
217                                log.error('Bad IP: %s', ex)
218                                continue
219
220                self.config[key] = list(saved)
221                continue
222            elif key == 'check_after_days':
223                if self.config[key] != config[key]:
224                    self.config[key] = config[key]
225                    update_now = False
226                    if self.config['last_update']:
227                        last_update = datetime.fromtimestamp(self.config['last_update'])
228                        check_period = timedelta(days=self.config['check_after_days'])
229                    if (
230                        not self.config['last_update']
231                        or last_update + check_period < datetime.now()
232                    ):
233                        update_now = True
234                    if self.update_timer.running:
235                        self.update_timer.stop()
236                    if self.config['check_after_days'] > 0:
237                        self.update_timer.start(
238                            self.config['check_after_days'] * 24 * 60 * 60, update_now
239                        )
240                continue
241            self.config[key] = config[key]
242
243        if needs_blocklist_import:
244            log.debug(
245                'IP addresses were removed from the whitelist. Since we '
246                'do not know if they were blocked before. Re-import '
247                'current blocklist and re-add whitelisted.'
248            )
249            self.has_imported = False
250            d = self.import_list(deluge.configmanager.get_config_dir('blocklist.cache'))
251            d.addCallbacks(self.on_import_complete, self.on_import_error)
252
253    @export
254    def get_status(self):
255        """Get the status of the plugin.
256
257        Returns:
258            dict: The status dict of the plugin.
259
260        """
261        status = {}
262        if self.is_downloading:
263            status['state'] = 'Downloading'
264        elif self.is_importing:
265            status['state'] = 'Importing'
266        else:
267            status['state'] = 'Idle'
268
269        status['up_to_date'] = self.up_to_date
270        status['num_whited'] = self.num_whited
271        status['num_blocked'] = self.num_blocked
272        status['file_progress'] = self.file_progress
273        status['file_url'] = self.config['url']
274        status['file_size'] = self.config['list_size']
275        status['file_date'] = self.config['last_update']
276        status['file_type'] = self.config['list_type']
277        status['whitelisted'] = self.config['whitelisted']
278        if self.config['list_compression']:
279            status['file_type'] += ' (%s)' % self.config['list_compression']
280        return status
281
282    ####
283
284    def update_info(self, blocklist):
285        """Updates blocklist info.
286
287        Args:
288            blocklist (str): Path of blocklist.
289
290        Returns:
291            str: Path of blocklist.
292
293        """
294        log.debug('Updating blocklist info: %s', blocklist)
295        self.config['last_update'] = time.time()
296        self.config['list_size'] = os.path.getsize(blocklist)
297        self.filename = blocklist
298        return blocklist
299
300    def download_list(self, url=None):
301        """Downloads the blocklist specified by 'url' in the config.
302
303        Args:
304            url (str, optional): url to download from, defaults to config value.
305
306        Returns:
307            Deferred: a Deferred which fires once the blocklist has been downloaded.
308
309        """
310
311        def on_retrieve_data(data, current_length, total_length):
312            if total_length:
313                fp = current_length / total_length
314                if fp > 1.0:
315                    fp = 1.0
316            else:
317                fp = 0.0
318
319            self.file_progress = fp
320
321        import socket
322
323        socket.setdefaulttimeout(self.config['timeout'])
324
325        if not url:
326            url = self.config['url']
327
328        headers = {}
329        if self.config['last_update'] and not self.force_download:
330            headers['If-Modified-Since'] = formatdate(
331                self.config['last_update'], usegmt=True
332            )
333
334        log.debug('Attempting to download blocklist %s', url)
335        log.debug('Sending headers: %s', headers)
336        self.is_downloading = True
337        return download_file(
338            url,
339            deluge.configmanager.get_config_dir('blocklist.download'),
340            on_retrieve_data,
341            headers,
342        )
343
344    def on_download_complete(self, blocklist):
345        """Runs any download clean up functions.
346
347        Args:
348            blocklist (str): Path of blocklist.
349
350        Returns:
351            Deferred: a Deferred which fires when clean up is done.
352
353        """
354        log.debug('Blocklist download complete: %s', blocklist)
355        self.is_downloading = False
356        return threads.deferToThread(self.update_info, blocklist)
357
358    def on_download_error(self, f):
359        """Recovers from download error.
360
361        Args:
362            f (Failure): Failure that occurred.
363
364        Returns:
365            Deferred or Failure: A Deferred if recovery was possible else original Failure.
366
367        """
368        self.is_downloading = False
369        error_msg = f.getErrorMessage()
370        d = f
371        if f.check(error.PageRedirect):
372            # Handle redirect errors
373            location = urljoin(self.config['url'], error_msg.split(' to ')[1])
374            if 'Moved Permanently' in error_msg:
375                log.debug('Setting blocklist url to %s', location)
376                self.config['url'] = location
377            d = self.download_list(location)
378            d.addCallbacks(self.on_download_complete, self.on_download_error)
379        else:
380            if 'Not Modified' in error_msg:
381                log.debug('Blocklist is up-to-date!')
382                self.up_to_date = True
383                blocklist = deluge.configmanager.get_config_dir('blocklist.cache')
384                d = threads.deferToThread(self.update_info, blocklist)
385            else:
386                log.warning('Blocklist download failed: %s', error_msg)
387                if self.failed_attempts < self.config['try_times']:
388                    log.debug(
389                        'Try downloading blocklist again... (%s/%s)',
390                        self.failed_attempts,
391                        self.config['try_times'],
392                    )
393                    self.failed_attempts += 1
394                    d = self.download_list()
395                    d.addCallbacks(self.on_download_complete, self.on_download_error)
396        return d
397
398    def import_list(self, blocklist):
399        """Imports the downloaded blocklist into the session.
400
401        Args:
402            blocklist (str): path of blocklist.
403
404        Returns:
405            Deferred: A Deferred that fires when the blocklist has been imported.
406
407        """
408        log.trace('on import_list')
409
410        def on_read_ip_range(start, end):
411            """Add ip range to blocklist"""
412            # log.trace('Adding ip range %s - %s to ipfilter as blocked', start, end)
413            self.blocklist.add_rule(start.address, end.address, BLOCK_RANGE)
414            self.num_blocked += 1
415
416        def on_finish_read(result):
417            """Add any whitelisted IP's and add the blocklist to session"""
418            # White listing happens last because the last rules added have
419            # priority
420            log.info('Added %d ranges to ipfilter as blocked', self.num_blocked)
421            for ip in self.config['whitelisted']:
422                ip = IP.parse(ip)
423                self.blocklist.add_rule(ip.address, ip.address, ALLOW_RANGE)
424                self.num_whited += 1
425                log.trace('Added %s to the ipfiler as white-listed', ip.address)
426            log.info('Added %d ranges to ipfilter as white-listed', self.num_whited)
427            self.core.session.set_ip_filter(self.blocklist)
428            return result
429
430        # TODO: double check logic
431        if self.up_to_date and self.has_imported:
432            log.debug('Latest blocklist is already imported')
433            return defer.succeed(blocklist)
434
435        self.is_importing = True
436        self.num_blocked = 0
437        self.num_whited = 0
438        self.blocklist = self.core.session.get_ip_filter()
439
440        if not blocklist:
441            blocklist = self.filename
442
443        if not self.reader:
444            self.auto_detect(blocklist)
445            self.auto_detected = True
446
447        def on_reader_failure(failure):
448            log.error('Failed to read!!!!!!')
449            log.exception(failure)
450
451        log.debug('Importing using reader: %s', self.reader)
452        log.debug(
453            'Reader type: %s compression: %s',
454            self.config['list_type'],
455            self.config['list_compression'],
456        )
457        log.debug('Clearing current ip filtering')
458        # self.blocklist.add_rule('0.0.0.0', '255.255.255.255', ALLOW_RANGE)
459        d = threads.deferToThread(self.reader(blocklist).read, on_read_ip_range)
460        d.addCallback(on_finish_read).addErrback(on_reader_failure)
461
462        return d
463
464    def on_import_complete(self, blocklist):
465        """Runs any import clean up functions.
466
467        Args:
468            blocklist (str): Path of blocklist.
469
470        Returns:
471            Deferred: A Deferred that fires when clean up is done.
472
473        """
474        log.trace('on_import_list_complete')
475        d = blocklist
476        self.is_importing = False
477        self.has_imported = True
478        log.debug('Blocklist import complete!')
479        cache = deluge.configmanager.get_config_dir('blocklist.cache')
480        if blocklist != cache:
481            if self.is_url:
482                log.debug('Moving %s to %s', blocklist, cache)
483                d = threads.deferToThread(shutil.move, blocklist, cache)
484            else:
485                log.debug('Copying %s to %s', blocklist, cache)
486                d = threads.deferToThread(shutil.copy, blocklist, cache)
487        return d
488
489    def on_import_error(self, f):
490        """Recovers from import error.
491
492        Args:
493            f (Failure): Failure that occurred.
494
495        Returns:
496            Deferred or Failure: A Deferred if recovery was possible else original Failure.
497
498        """
499        log.trace('on_import_error: %s', f)
500        d = f
501        self.is_importing = False
502        try_again = False
503        cache = deluge.configmanager.get_config_dir('blocklist.cache')
504
505        if f.check(ReaderParseError) and not self.auto_detected:
506            # Invalid / corrupt list, let's detect it
507            log.warning('Invalid / corrupt blocklist')
508            self.reader = None
509            blocklist = None
510            try_again = True
511        elif self.filename != cache and os.path.exists(cache):
512            # If we have a backup and we haven't already used it
513            log.warning('Error reading blocklist: %s', f.getErrorMessage())
514            blocklist = cache
515            try_again = True
516
517        if try_again:
518            d = self.import_list(blocklist)
519            d.addCallbacks(self.on_import_complete, self.on_import_error)
520
521        return d
522
523    def auto_detect(self, blocklist):
524        """Attempts to auto-detect the blocklist type.
525
526        Args:
527            blocklist (str): Path of blocklist.
528
529        Raises:
530            UnknownFormatError: If the format cannot be detected.
531
532        """
533        self.config['list_compression'] = detect_compression(blocklist)
534        self.config['list_type'] = detect_format(
535            blocklist, self.config['list_compression']
536        )
537        log.debug(
538            'Auto-detected type: %s compression: %s',
539            self.config['list_type'],
540            self.config['list_compression'],
541        )
542        if not self.config['list_type']:
543            self.config['list_compression'] = ''
544            raise UnknownFormatError
545        else:
546            self.reader = create_reader(
547                self.config['list_type'], self.config['list_compression']
548            )
549
550    def pause_session(self):
551        self.need_to_resume_session = not self.core.session.is_paused()
552        self.core.pause_session()
553
554    def resume_session(self, result):
555        self.core.resume_session()
556        self.need_to_resume_session = False
557        return result
558