1from loguru import logger 2 3from flexget import plugin 4from flexget.event import event 5 6from . import db 7 8logger = logger.bind(name='seen') 9 10 11class FilterSeen: 12 """ 13 Remembers previously downloaded content and rejects them in 14 subsequent executions. Without this plugin FlexGet would 15 download all matching content on every execution. 16 17 This plugin is enabled on all tasks by default. 18 See wiki for more information. 19 """ 20 21 schema = { 22 'oneOf': [ 23 {'type': 'boolean'}, 24 {'type': 'string', 'enum': ['global', 'local']}, 25 { 26 'type': 'object', 27 'properties': { 28 'local': {'type': 'boolean'}, 29 'fields': { 30 'type': 'array', 31 'items': {'type': 'string'}, 32 "minItems": 1, 33 "uniqueItems": True, 34 }, 35 }, 36 }, 37 ] 38 } 39 40 def __init__(self): 41 # remember and filter by these fields 42 self.fields = ['title', 'url', 'original_url'] 43 self.keyword = 'seen' 44 45 def prepare_config(self, config): 46 if config is None: 47 config = {} 48 elif isinstance(config, bool): 49 if config is False: 50 return config 51 else: 52 config = {'local': False} 53 elif isinstance(config, str): 54 config = {'local': config == 'local'} 55 56 config.setdefault('local', False) 57 config.setdefault('fields', self.fields) 58 return config 59 60 @plugin.priority(plugin.PRIORITY_FIRST) 61 def on_task_filter(self, task, config, remember_rejected=False): 62 """Filter entries already accepted on previous runs.""" 63 config = self.prepare_config(config) 64 if config is False: 65 logger.debug('{} is disabled', self.keyword) 66 return 67 68 fields = config.get('fields') 69 local = config.get('local') 70 71 for entry in task.entries: 72 # construct list of values looked 73 values = [] 74 for field in fields: 75 if field not in entry: 76 continue 77 if entry[field] not in values and entry[field]: 78 values.append(str(entry[field])) 79 if values: 80 logger.trace('querying for: {}', ', '.join(values)) 81 # check if SeenField.value is any of the values 82 found = db.search_by_field_values( 83 field_value_list=values, task_name=task.name, local=local, session=task.session 84 ) 85 if found: 86 logger.debug( 87 "Rejecting '{}' '{}' because of seen '{}'", 88 entry['url'], 89 entry['title'], 90 found.value, 91 ) 92 se = ( 93 task.session.query(db.SeenEntry) 94 .filter(db.SeenEntry.id == found.seen_entry_id) 95 .one() 96 ) 97 entry.reject( 98 'Entry with %s `%s` is already marked seen in the task %s at %s' 99 % (found.field, found.value, se.task, se.added.strftime('%Y-%m-%d %H:%M')), 100 remember=remember_rejected, 101 ) 102 103 def on_task_learn(self, task, config): 104 """Remember succeeded entries""" 105 config = self.prepare_config(config) 106 if config is False: 107 logger.debug('disabled') 108 return 109 110 fields = config.get('fields') 111 local = config.get('local') 112 113 if isinstance(config, list): 114 fields.extend(config) 115 116 for entry in task.accepted: 117 self.learn(task, entry, fields=fields, local=local) 118 # verbose if in learning mode 119 if task.options.learn: 120 logger.info("Learned '{}' (will skip this in the future)", entry['title']) 121 122 def learn(self, task, entry, fields=None, reason=None, local=False): 123 """Marks entry as seen""" 124 # no explicit fields given, use default 125 if not fields: 126 fields = self.fields 127 se = db.SeenEntry(entry['title'], str(task.name), reason, local) 128 remembered = [] 129 for field in fields: 130 if field not in entry: 131 continue 132 # removes duplicate values (eg. url, original_url are usually same) 133 if entry[field] in remembered: 134 continue 135 remembered.append(entry[field]) 136 sf = db.SeenField(str(field), str(entry[field])) 137 se.fields.append(sf) 138 logger.debug("Learned '{}' (field: {}, local: {})", entry[field], field, local) 139 # Only add the entry to the session if it has one of the required fields 140 if se.fields: 141 task.session.add(se) 142 143 def forget(self, task, title): 144 """Forget SeenEntry with :title:. Return True if forgotten.""" 145 se = task.session.query(db.SeenEntry).filter(db.SeenEntry.title == title).first() 146 if se: 147 logger.debug("Forgotten '{}' ({} fields)", title, len(se.fields)) 148 task.session.delete(se) 149 return True 150 151 152@event('plugin.register') 153def register_plugin(): 154 plugin.register(FilterSeen, 'seen', builtin=True, api_ver=2) 155