1# -*- coding: utf-8 -*- 2# 3# Copyright (C) 2008 Andrew Resch <andrewresch@gmail.com> 4# Copyright (C) 2009-2010 John Garland <johnnybg+deluge@gmail.com> 5# 6# This file is part of Deluge and is licensed under GNU General Public License 3.0, or later, with 7# the additional special exception to link portions of this program with the OpenSSL library. 8# See LICENSE for more details. 9# 10 11from __future__ import division, unicode_literals 12 13import logging 14import os 15import shutil 16import time 17from datetime import datetime, timedelta 18from email.utils import formatdate 19 20from twisted.internet import defer, threads 21from twisted.internet.task import LoopingCall 22from twisted.web import error 23 24import deluge.component as component 25import deluge.configmanager 26from deluge.common import is_url 27from deluge.core.rpcserver import export 28from deluge.httpdownloader import download_file 29from deluge.plugins.pluginbase import CorePluginBase 30 31from .common import IP, BadIP 32from .detect import UnknownFormatError, create_reader, detect_compression, detect_format 33from .readers import ReaderParseError 34 35try: 36 from urllib.parse import urljoin 37except ImportError: 38 # PY2 fallback 39 from urlparse import urljoin # pylint: disable=ungrouped-imports 40 41# TODO: check return values for deferred callbacks 42# TODO: review class attributes for redundancy 43 44log = logging.getLogger(__name__) 45 46DEFAULT_PREFS = { 47 'url': '', 48 'load_on_start': False, 49 'check_after_days': 4, 50 'list_compression': '', 51 'list_type': '', 52 'last_update': 0.0, 53 'list_size': 0, 54 'timeout': 180, 55 'try_times': 3, 56 'whitelisted': [], 57} 58 59# Constants 60ALLOW_RANGE = 0 61BLOCK_RANGE = 1 62 63 64class Core(CorePluginBase): 65 def enable(self): 66 log.debug('Blocklist: Plugin enabled...') 67 68 self.is_url = True 69 self.is_downloading = False 70 self.is_importing = False 71 self.has_imported = False 72 self.up_to_date = False 73 self.need_to_resume_session = False 74 self.num_whited = 0 75 self.num_blocked = 0 76 self.file_progress = 0.0 77 78 self.core = component.get('Core') 79 self.config = deluge.configmanager.ConfigManager( 80 'blocklist.conf', DEFAULT_PREFS 81 ) 82 if 'whitelisted' not in self.config: 83 self.config['whitelisted'] = [] 84 85 self.reader = create_reader( 86 self.config['list_type'], self.config['list_compression'] 87 ) 88 89 if not isinstance(self.config['last_update'], float): 90 self.config.config['last_update'] = 0.0 91 92 update_now = False 93 if self.config['load_on_start']: 94 self.pause_session() 95 if self.config['last_update']: 96 last_update = datetime.fromtimestamp(self.config['last_update']) 97 check_period = timedelta(days=self.config['check_after_days']) 98 if ( 99 not self.config['last_update'] 100 or last_update + check_period < datetime.now() 101 ): 102 update_now = True 103 else: 104 d = self.import_list( 105 deluge.configmanager.get_config_dir('blocklist.cache') 106 ) 107 d.addCallbacks(self.on_import_complete, self.on_import_error) 108 if self.need_to_resume_session: 109 d.addBoth(self.resume_session) 110 111 # This function is called every 'check_after_days' days, to download 112 # and import a new list if needed. 113 self.update_timer = LoopingCall(self.check_import) 114 if self.config['check_after_days'] > 0: 115 self.update_timer.start( 116 self.config['check_after_days'] * 24 * 60 * 60, update_now 117 ) 118 119 def disable(self): 120 self.config.save() 121 log.debug('Reset IP filter') 122 self.core.session.get_ip_filter().add_rule( 123 '0.0.0.0', '255.255.255.255', ALLOW_RANGE 124 ) 125 log.debug('Blocklist: Plugin disabled') 126 127 def update(self): 128 pass 129 130 # Exported RPC methods # 131 @export 132 def check_import(self, force=False): 133 """Imports latest blocklist specified by blocklist url. 134 135 Args: 136 force (bool, optional): Force the download/import, default is False. 137 138 Returns: 139 Deferred: A Deferred which fires when the blocklist has been imported. 140 141 """ 142 if not self.config['url']: 143 return 144 145 # Reset variables 146 self.filename = None 147 self.force_download = force 148 self.failed_attempts = 0 149 self.auto_detected = False 150 self.up_to_date = False 151 if force: 152 self.reader = None 153 self.is_url = is_url(self.config['url']) 154 155 # Start callback chain 156 if self.is_url: 157 d = self.download_list() 158 d.addCallbacks(self.on_download_complete, self.on_download_error) 159 d.addCallback(self.import_list) 160 else: 161 d = self.import_list(self.config['url']) 162 d.addCallbacks(self.on_import_complete, self.on_import_error) 163 if self.need_to_resume_session: 164 d.addBoth(self.resume_session) 165 166 return d 167 168 @export 169 def get_config(self): 170 """Gets the blocklist config dictionary. 171 172 Returns: 173 dict: The config dictionary. 174 175 """ 176 return self.config.config 177 178 @export 179 def set_config(self, config): 180 """Sets the blocklist config. 181 182 Args: 183 config (dict): config to set. 184 185 """ 186 needs_blocklist_import = False 187 for key in config: 188 if key == 'whitelisted': 189 saved = set(self.config[key]) 190 update = set(config[key]) 191 diff = saved.symmetric_difference(update) 192 if diff: 193 log.debug('Whitelist changed. Updating...') 194 added = update.intersection(diff) 195 removed = saved.intersection(diff) 196 if added: 197 for ip in added: 198 try: 199 ip = IP.parse(ip) 200 self.blocklist.add_rule( 201 ip.address, ip.address, ALLOW_RANGE 202 ) 203 saved.add(ip.address) 204 log.debug('Added %s to whitelisted', ip) 205 self.num_whited += 1 206 except BadIP as ex: 207 log.error('Bad IP: %s', ex) 208 continue 209 if removed: 210 needs_blocklist_import = True 211 for ip in removed: 212 try: 213 ip = IP.parse(ip) 214 saved.remove(ip.address) 215 log.debug('Removed %s from whitelisted', ip) 216 except BadIP as ex: 217 log.error('Bad IP: %s', ex) 218 continue 219 220 self.config[key] = list(saved) 221 continue 222 elif key == 'check_after_days': 223 if self.config[key] != config[key]: 224 self.config[key] = config[key] 225 update_now = False 226 if self.config['last_update']: 227 last_update = datetime.fromtimestamp(self.config['last_update']) 228 check_period = timedelta(days=self.config['check_after_days']) 229 if ( 230 not self.config['last_update'] 231 or last_update + check_period < datetime.now() 232 ): 233 update_now = True 234 if self.update_timer.running: 235 self.update_timer.stop() 236 if self.config['check_after_days'] > 0: 237 self.update_timer.start( 238 self.config['check_after_days'] * 24 * 60 * 60, update_now 239 ) 240 continue 241 self.config[key] = config[key] 242 243 if needs_blocklist_import: 244 log.debug( 245 'IP addresses were removed from the whitelist. Since we ' 246 'do not know if they were blocked before. Re-import ' 247 'current blocklist and re-add whitelisted.' 248 ) 249 self.has_imported = False 250 d = self.import_list(deluge.configmanager.get_config_dir('blocklist.cache')) 251 d.addCallbacks(self.on_import_complete, self.on_import_error) 252 253 @export 254 def get_status(self): 255 """Get the status of the plugin. 256 257 Returns: 258 dict: The status dict of the plugin. 259 260 """ 261 status = {} 262 if self.is_downloading: 263 status['state'] = 'Downloading' 264 elif self.is_importing: 265 status['state'] = 'Importing' 266 else: 267 status['state'] = 'Idle' 268 269 status['up_to_date'] = self.up_to_date 270 status['num_whited'] = self.num_whited 271 status['num_blocked'] = self.num_blocked 272 status['file_progress'] = self.file_progress 273 status['file_url'] = self.config['url'] 274 status['file_size'] = self.config['list_size'] 275 status['file_date'] = self.config['last_update'] 276 status['file_type'] = self.config['list_type'] 277 status['whitelisted'] = self.config['whitelisted'] 278 if self.config['list_compression']: 279 status['file_type'] += ' (%s)' % self.config['list_compression'] 280 return status 281 282 #### 283 284 def update_info(self, blocklist): 285 """Updates blocklist info. 286 287 Args: 288 blocklist (str): Path of blocklist. 289 290 Returns: 291 str: Path of blocklist. 292 293 """ 294 log.debug('Updating blocklist info: %s', blocklist) 295 self.config['last_update'] = time.time() 296 self.config['list_size'] = os.path.getsize(blocklist) 297 self.filename = blocklist 298 return blocklist 299 300 def download_list(self, url=None): 301 """Downloads the blocklist specified by 'url' in the config. 302 303 Args: 304 url (str, optional): url to download from, defaults to config value. 305 306 Returns: 307 Deferred: a Deferred which fires once the blocklist has been downloaded. 308 309 """ 310 311 def on_retrieve_data(data, current_length, total_length): 312 if total_length: 313 fp = current_length / total_length 314 if fp > 1.0: 315 fp = 1.0 316 else: 317 fp = 0.0 318 319 self.file_progress = fp 320 321 import socket 322 323 socket.setdefaulttimeout(self.config['timeout']) 324 325 if not url: 326 url = self.config['url'] 327 328 headers = {} 329 if self.config['last_update'] and not self.force_download: 330 headers['If-Modified-Since'] = formatdate( 331 self.config['last_update'], usegmt=True 332 ) 333 334 log.debug('Attempting to download blocklist %s', url) 335 log.debug('Sending headers: %s', headers) 336 self.is_downloading = True 337 return download_file( 338 url, 339 deluge.configmanager.get_config_dir('blocklist.download'), 340 on_retrieve_data, 341 headers, 342 ) 343 344 def on_download_complete(self, blocklist): 345 """Runs any download clean up functions. 346 347 Args: 348 blocklist (str): Path of blocklist. 349 350 Returns: 351 Deferred: a Deferred which fires when clean up is done. 352 353 """ 354 log.debug('Blocklist download complete: %s', blocklist) 355 self.is_downloading = False 356 return threads.deferToThread(self.update_info, blocklist) 357 358 def on_download_error(self, f): 359 """Recovers from download error. 360 361 Args: 362 f (Failure): Failure that occurred. 363 364 Returns: 365 Deferred or Failure: A Deferred if recovery was possible else original Failure. 366 367 """ 368 self.is_downloading = False 369 error_msg = f.getErrorMessage() 370 d = f 371 if f.check(error.PageRedirect): 372 # Handle redirect errors 373 location = urljoin(self.config['url'], error_msg.split(' to ')[1]) 374 if 'Moved Permanently' in error_msg: 375 log.debug('Setting blocklist url to %s', location) 376 self.config['url'] = location 377 d = self.download_list(location) 378 d.addCallbacks(self.on_download_complete, self.on_download_error) 379 else: 380 if 'Not Modified' in error_msg: 381 log.debug('Blocklist is up-to-date!') 382 self.up_to_date = True 383 blocklist = deluge.configmanager.get_config_dir('blocklist.cache') 384 d = threads.deferToThread(self.update_info, blocklist) 385 else: 386 log.warning('Blocklist download failed: %s', error_msg) 387 if self.failed_attempts < self.config['try_times']: 388 log.debug( 389 'Try downloading blocklist again... (%s/%s)', 390 self.failed_attempts, 391 self.config['try_times'], 392 ) 393 self.failed_attempts += 1 394 d = self.download_list() 395 d.addCallbacks(self.on_download_complete, self.on_download_error) 396 return d 397 398 def import_list(self, blocklist): 399 """Imports the downloaded blocklist into the session. 400 401 Args: 402 blocklist (str): path of blocklist. 403 404 Returns: 405 Deferred: A Deferred that fires when the blocklist has been imported. 406 407 """ 408 log.trace('on import_list') 409 410 def on_read_ip_range(start, end): 411 """Add ip range to blocklist""" 412 # log.trace('Adding ip range %s - %s to ipfilter as blocked', start, end) 413 self.blocklist.add_rule(start.address, end.address, BLOCK_RANGE) 414 self.num_blocked += 1 415 416 def on_finish_read(result): 417 """Add any whitelisted IP's and add the blocklist to session""" 418 # White listing happens last because the last rules added have 419 # priority 420 log.info('Added %d ranges to ipfilter as blocked', self.num_blocked) 421 for ip in self.config['whitelisted']: 422 ip = IP.parse(ip) 423 self.blocklist.add_rule(ip.address, ip.address, ALLOW_RANGE) 424 self.num_whited += 1 425 log.trace('Added %s to the ipfiler as white-listed', ip.address) 426 log.info('Added %d ranges to ipfilter as white-listed', self.num_whited) 427 self.core.session.set_ip_filter(self.blocklist) 428 return result 429 430 # TODO: double check logic 431 if self.up_to_date and self.has_imported: 432 log.debug('Latest blocklist is already imported') 433 return defer.succeed(blocklist) 434 435 self.is_importing = True 436 self.num_blocked = 0 437 self.num_whited = 0 438 self.blocklist = self.core.session.get_ip_filter() 439 440 if not blocklist: 441 blocklist = self.filename 442 443 if not self.reader: 444 self.auto_detect(blocklist) 445 self.auto_detected = True 446 447 def on_reader_failure(failure): 448 log.error('Failed to read!!!!!!') 449 log.exception(failure) 450 451 log.debug('Importing using reader: %s', self.reader) 452 log.debug( 453 'Reader type: %s compression: %s', 454 self.config['list_type'], 455 self.config['list_compression'], 456 ) 457 log.debug('Clearing current ip filtering') 458 # self.blocklist.add_rule('0.0.0.0', '255.255.255.255', ALLOW_RANGE) 459 d = threads.deferToThread(self.reader(blocklist).read, on_read_ip_range) 460 d.addCallback(on_finish_read).addErrback(on_reader_failure) 461 462 return d 463 464 def on_import_complete(self, blocklist): 465 """Runs any import clean up functions. 466 467 Args: 468 blocklist (str): Path of blocklist. 469 470 Returns: 471 Deferred: A Deferred that fires when clean up is done. 472 473 """ 474 log.trace('on_import_list_complete') 475 d = blocklist 476 self.is_importing = False 477 self.has_imported = True 478 log.debug('Blocklist import complete!') 479 cache = deluge.configmanager.get_config_dir('blocklist.cache') 480 if blocklist != cache: 481 if self.is_url: 482 log.debug('Moving %s to %s', blocklist, cache) 483 d = threads.deferToThread(shutil.move, blocklist, cache) 484 else: 485 log.debug('Copying %s to %s', blocklist, cache) 486 d = threads.deferToThread(shutil.copy, blocklist, cache) 487 return d 488 489 def on_import_error(self, f): 490 """Recovers from import error. 491 492 Args: 493 f (Failure): Failure that occurred. 494 495 Returns: 496 Deferred or Failure: A Deferred if recovery was possible else original Failure. 497 498 """ 499 log.trace('on_import_error: %s', f) 500 d = f 501 self.is_importing = False 502 try_again = False 503 cache = deluge.configmanager.get_config_dir('blocklist.cache') 504 505 if f.check(ReaderParseError) and not self.auto_detected: 506 # Invalid / corrupt list, let's detect it 507 log.warning('Invalid / corrupt blocklist') 508 self.reader = None 509 blocklist = None 510 try_again = True 511 elif self.filename != cache and os.path.exists(cache): 512 # If we have a backup and we haven't already used it 513 log.warning('Error reading blocklist: %s', f.getErrorMessage()) 514 blocklist = cache 515 try_again = True 516 517 if try_again: 518 d = self.import_list(blocklist) 519 d.addCallbacks(self.on_import_complete, self.on_import_error) 520 521 return d 522 523 def auto_detect(self, blocklist): 524 """Attempts to auto-detect the blocklist type. 525 526 Args: 527 blocklist (str): Path of blocklist. 528 529 Raises: 530 UnknownFormatError: If the format cannot be detected. 531 532 """ 533 self.config['list_compression'] = detect_compression(blocklist) 534 self.config['list_type'] = detect_format( 535 blocklist, self.config['list_compression'] 536 ) 537 log.debug( 538 'Auto-detected type: %s compression: %s', 539 self.config['list_type'], 540 self.config['list_compression'], 541 ) 542 if not self.config['list_type']: 543 self.config['list_compression'] = '' 544 raise UnknownFormatError 545 else: 546 self.reader = create_reader( 547 self.config['list_type'], self.config['list_compression'] 548 ) 549 550 def pause_session(self): 551 self.need_to_resume_session = not self.core.session.is_paused() 552 self.core.pause_session() 553 554 def resume_session(self, result): 555 self.core.resume_session() 556 self.need_to_resume_session = False 557 return result 558