1#!/usr/local/bin/python3.8 2# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai 3 4 5__license__ = 'GPL v3' 6__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' 7__docformat__ = 'restructuredtext en' 8 9import weakref, operator, numbers 10from functools import partial 11from polyglot.builtins import iteritems, itervalues 12 13from calibre.ebooks.metadata import title_sort 14from calibre.utils.config_base import tweaks, prefs 15from calibre.db.write import uniq 16 17 18def sanitize_sort_field_name(field_metadata, field): 19 field = field_metadata.search_term_to_field_key(field.lower().strip()) 20 # translate some fields to their hidden equivalent 21 field = {'title': 'sort', 'authors':'author_sort'}.get(field, field) 22 return field 23 24 25class MarkedVirtualField: 26 27 def __init__(self, marked_ids): 28 self.marked_ids = marked_ids 29 30 def iter_searchable_values(self, get_metadata, candidates, default_value=None): 31 for book_id in candidates: 32 yield self.marked_ids.get(book_id, default_value), {book_id} 33 34 def sort_keys_for_books(self, get_metadata, lang_map): 35 g = self.marked_ids.get 36 return lambda book_id:g(book_id, '') 37 38 39class TableRow: 40 41 def __init__(self, book_id, view): 42 self.book_id = book_id 43 self.view = weakref.ref(view) 44 self.column_count = view.column_count 45 46 def __getitem__(self, obj): 47 view = self.view() 48 if isinstance(obj, slice): 49 return [view._field_getters[c](self.book_id) 50 for c in range(*obj.indices(len(view._field_getters)))] 51 else: 52 return view._field_getters[obj](self.book_id) 53 54 def __len__(self): 55 return self.column_count 56 57 def __iter__(self): 58 for i in range(self.column_count): 59 yield self[i] 60 61 62def format_is_multiple(x, sep=',', repl=None): 63 if not x: 64 return None 65 if repl is not None: 66 x = (y.replace(sep, repl) for y in x) 67 return sep.join(x) 68 69 70def format_identifiers(x): 71 if not x: 72 return None 73 return ','.join('%s:%s'%(k, v) for k, v in iteritems(x)) 74 75 76class View: 77 78 ''' A table view of the database, with rows and columns. Also supports 79 filtering and sorting. ''' 80 81 def __init__(self, cache): 82 self.cache = cache 83 self.marked_ids = {} 84 self.marked_listeners = {} 85 self.search_restriction_book_count = 0 86 self.search_restriction = self.base_restriction = '' 87 self.search_restriction_name = self.base_restriction_name = '' 88 self._field_getters = {} 89 self.column_count = len(cache.backend.FIELD_MAP) 90 for col, idx in iteritems(cache.backend.FIELD_MAP): 91 label, fmt = col, lambda x:x 92 func = { 93 'id': self._get_id, 94 'au_map': self.get_author_data, 95 'ondevice': self.get_ondevice, 96 'marked': self.get_marked, 97 'series_sort':self.get_series_sort, 98 }.get(col, self._get) 99 if isinstance(col, numbers.Integral): 100 label = self.cache.backend.custom_column_num_map[col]['label'] 101 label = (self.cache.backend.field_metadata.custom_field_prefix + label) 102 if label.endswith('_index'): 103 try: 104 num = int(label.partition('_')[0]) 105 except ValueError: 106 pass # series_index 107 else: 108 label = self.cache.backend.custom_column_num_map[num]['label'] 109 label = (self.cache.backend.field_metadata.custom_field_prefix + label + '_index') 110 111 fm = self.field_metadata[label] 112 fm 113 if label == 'authors': 114 fmt = partial(format_is_multiple, repl='|') 115 elif label in {'tags', 'languages', 'formats'}: 116 fmt = format_is_multiple 117 elif label == 'cover': 118 fmt = bool 119 elif label == 'identifiers': 120 fmt = format_identifiers 121 elif fm['datatype'] == 'text' and fm['is_multiple']: 122 sep = fm['is_multiple']['cache_to_list'] 123 if sep not in {'&','|'}: 124 sep = '|' 125 fmt = partial(format_is_multiple, sep=sep) 126 self._field_getters[idx] = partial(func, label, fmt=fmt) if func == self._get else func 127 128 self._map = tuple(sorted(self.cache.all_book_ids())) 129 self._map_filtered = tuple(self._map) 130 self.full_map_is_sorted = True 131 self.sort_history = [('id', True)] 132 133 def add_marked_listener(self, func): 134 self.marked_listeners[id(func)] = weakref.ref(func) 135 136 def add_to_sort_history(self, items): 137 self.sort_history = uniq((list(items) + list(self.sort_history)), 138 operator.itemgetter(0))[:tweaks['maximum_resort_levels']] 139 140 def count(self): 141 return len(self._map) 142 143 def get_property(self, id_or_index, index_is_id=False, loc=-1): 144 book_id = id_or_index if index_is_id else self._map_filtered[id_or_index] 145 return self._field_getters[loc](book_id) 146 147 def sanitize_sort_field_name(self, field): 148 return sanitize_sort_field_name(self.field_metadata, field) 149 150 @property 151 def field_metadata(self): 152 return self.cache.field_metadata 153 154 def _get_id(self, idx, index_is_id=True): 155 if index_is_id and not self.cache.has_id(idx): 156 raise IndexError('No book with id %s present'%idx) 157 return idx if index_is_id else self.index_to_id(idx) 158 159 def has_id(self, book_id): 160 return self.cache.has_id(book_id) 161 162 def __getitem__(self, row): 163 return TableRow(self._map_filtered[row], self) 164 165 def __len__(self): 166 return len(self._map_filtered) 167 168 def __iter__(self): 169 for book_id in self._map_filtered: 170 yield TableRow(book_id, self) 171 172 def iterall(self): 173 for book_id in self.iterallids(): 174 yield TableRow(book_id, self) 175 176 def iterallids(self): 177 yield from sorted(self._map) 178 179 def tablerow_for_id(self, book_id): 180 return TableRow(book_id, self) 181 182 def get_field_map_field(self, row, col, index_is_id=True): 183 ''' 184 Supports the legacy FIELD_MAP interface for getting metadata. Do not use 185 in new code. 186 ''' 187 getter = self._field_getters[col] 188 return getter(row, index_is_id=index_is_id) 189 190 def index_to_id(self, idx): 191 return self._map_filtered[idx] 192 193 def id_to_index(self, book_id): 194 return self._map_filtered.index(book_id) 195 row = index_to_id 196 197 def index(self, book_id, cache=False): 198 x = self._map if cache else self._map_filtered 199 return x.index(book_id) 200 201 def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): 202 id_ = idx if index_is_id else self.index_to_id(idx) 203 if index_is_id and not self.cache.has_id(id_): 204 raise IndexError('No book with id %s present'%idx) 205 return fmt(self.cache.field_for(field, id_, default_value=default_value)) 206 207 def get_series_sort(self, idx, index_is_id=True, default_value=''): 208 book_id = idx if index_is_id else self.index_to_id(idx) 209 with self.cache.safe_read_lock: 210 lang_map = self.cache.fields['languages'].book_value_map 211 lang = lang_map.get(book_id, None) or None 212 if lang: 213 lang = lang[0] 214 return title_sort(self.cache._field_for('series', book_id, default_value=''), 215 order=tweaks['title_series_sorting'], lang=lang) 216 217 def get_ondevice(self, idx, index_is_id=True, default_value=''): 218 id_ = idx if index_is_id else self.index_to_id(idx) 219 return self.cache.field_for('ondevice', id_, default_value=default_value) 220 221 def get_marked(self, idx, index_is_id=True, default_value=None): 222 id_ = idx if index_is_id else self.index_to_id(idx) 223 return self.marked_ids.get(id_, default_value) 224 225 def get_author_data(self, idx, index_is_id=True, default_value=None): 226 id_ = idx if index_is_id else self.index_to_id(idx) 227 with self.cache.safe_read_lock: 228 ids = self.cache._field_ids_for('authors', id_) 229 adata = self.cache._author_data(ids) 230 ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata] 231 return ':#:'.join(ans) if ans else default_value 232 233 def get_virtual_libraries_for_books(self, ids): 234 return self.cache.virtual_libraries_for_books( 235 ids, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) 236 237 def _do_sort(self, ids_to_sort, fields=(), subsort=False): 238 fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields] 239 keys = self.field_metadata.sortable_field_keys() 240 fields = [x for x in fields if x[0] in keys] 241 if subsort and 'sort' not in [x[0] for x in fields]: 242 fields += [('sort', True)] 243 if not fields: 244 fields = [('timestamp', False)] 245 246 return self.cache.multisort( 247 fields, ids_to_sort=ids_to_sort, 248 virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) 249 250 def multisort(self, fields=[], subsort=False, only_ids=None): 251 sorted_book_ids = self._do_sort(self._map if only_ids is None else only_ids, fields=fields, subsort=subsort) 252 if only_ids is None: 253 self._map = tuple(sorted_book_ids) 254 self.full_map_is_sorted = True 255 self.add_to_sort_history(fields) 256 if len(self._map_filtered) == len(self._map): 257 self._map_filtered = tuple(self._map) 258 else: 259 fids = frozenset(self._map_filtered) 260 self._map_filtered = tuple(i for i in self._map if i in fids) 261 else: 262 smap = {book_id:i for i, book_id in enumerate(sorted_book_ids)} 263 only_ids.sort(key=smap.get) 264 265 def incremental_sort(self, fields=(), subsort=False): 266 if len(self._map) == len(self._map_filtered): 267 return self.multisort(fields=fields, subsort=subsort) 268 self._map_filtered = tuple(self._do_sort(self._map_filtered, fields=fields, subsort=subsort)) 269 self.full_map_is_sorted = False 270 self.add_to_sort_history(fields) 271 272 def search(self, query, return_matches=False, sort_results=True): 273 ans = self.search_getting_ids(query, self.search_restriction, 274 set_restriction_count=True, sort_results=sort_results) 275 if return_matches: 276 return ans 277 self._map_filtered = tuple(ans) 278 279 def _build_restriction_string(self, restriction): 280 if self.base_restriction: 281 if restriction: 282 return '(%s) and (%s)' % (self.base_restriction, restriction) 283 else: 284 return self.base_restriction 285 else: 286 return restriction 287 288 def search_getting_ids(self, query, search_restriction, 289 set_restriction_count=False, use_virtual_library=True, sort_results=True): 290 if use_virtual_library: 291 search_restriction = self._build_restriction_string(search_restriction) 292 q = '' 293 if not query or not query.strip(): 294 q = search_restriction 295 else: 296 q = query 297 if search_restriction: 298 q = '(%s) and (%s)' % (search_restriction, query) 299 if not q: 300 if set_restriction_count: 301 self.search_restriction_book_count = len(self._map) 302 rv = list(self._map) 303 if sort_results and not self.full_map_is_sorted: 304 rv = self._do_sort(rv, fields=self.sort_history) 305 self._map = tuple(rv) 306 self.full_map_is_sorted = True 307 return rv 308 matches = self.cache.search( 309 query, search_restriction, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) 310 if len(matches) == len(self._map): 311 rv = list(self._map) 312 else: 313 rv = [x for x in self._map if x in matches] 314 if sort_results and not self.full_map_is_sorted: 315 # We need to sort the search results 316 if matches.issubset(frozenset(self._map_filtered)): 317 rv = [x for x in self._map_filtered if x in matches] 318 else: 319 rv = self._do_sort(rv, fields=self.sort_history) 320 if len(matches) == len(self._map): 321 # We have sorted all ids, update self._map 322 self._map = tuple(rv) 323 self.full_map_is_sorted = True 324 if set_restriction_count and q == search_restriction: 325 self.search_restriction_book_count = len(rv) 326 return rv 327 328 def get_search_restriction(self): 329 return self.search_restriction 330 331 def set_search_restriction(self, s): 332 self.search_restriction = s 333 334 def get_base_restriction(self): 335 return self.base_restriction 336 337 def set_base_restriction(self, s): 338 self.base_restriction = s 339 340 def get_base_restriction_name(self): 341 return self.base_restriction_name 342 343 def set_base_restriction_name(self, s): 344 self.base_restriction_name = s 345 346 def get_search_restriction_name(self): 347 return self.search_restriction_name 348 349 def set_search_restriction_name(self, s): 350 self.search_restriction_name = s 351 352 def search_restriction_applied(self): 353 return bool(self.search_restriction) or bool(self.base_restriction) 354 355 def get_search_restriction_book_count(self): 356 return self.search_restriction_book_count 357 358 def change_search_locations(self, newlocs): 359 self.cache.change_search_locations(newlocs) 360 361 def set_marked_ids(self, id_dict): 362 ''' 363 ids in id_dict are "marked". They can be searched for by 364 using the search term ``marked:true``. Pass in an empty dictionary or 365 set to clear marked ids. 366 367 :param id_dict: Either a dictionary mapping ids to values or a set 368 of ids. In the latter case, the value is set to 'true' for all ids. If 369 a mapping is provided, then the search can be used to search for 370 particular values: ``marked:value`` 371 ''' 372 old_marked_ids = set(self.marked_ids) 373 if not hasattr(id_dict, 'items'): 374 # Simple list. Make it a dict of string 'true' 375 self.marked_ids = dict.fromkeys(id_dict, 'true') 376 else: 377 # Ensure that all the items in the dict are text 378 self.marked_ids = {k: str(v) for k, v in iteritems(id_dict)} 379 # This invalidates all searches in the cache even though the cache may 380 # be shared by multiple views. This is not ideal, but... 381 cmids = set(self.marked_ids) 382 changed_ids = old_marked_ids | cmids 383 self.cache.clear_search_caches(changed_ids) 384 self.cache.clear_caches(book_ids=changed_ids) 385 if old_marked_ids != cmids: 386 for funcref in itervalues(self.marked_listeners): 387 func = funcref() 388 if func is not None: 389 func(old_marked_ids, cmids) 390 391 def toggle_marked_ids(self, book_ids): 392 book_ids = set(book_ids) 393 mids = set(self.marked_ids) 394 common = mids.intersection(book_ids) 395 self.set_marked_ids((mids | book_ids) - common) 396 397 def refresh(self, field=None, ascending=True, clear_caches=True, do_search=True): 398 self._map = tuple(sorted(self.cache.all_book_ids())) 399 self._map_filtered = tuple(self._map) 400 self.full_map_is_sorted = True 401 self.sort_history = [('id', True)] 402 if clear_caches: 403 self.cache.clear_caches() 404 if field is not None: 405 self.sort(field, ascending) 406 if do_search and (self.search_restriction or self.base_restriction): 407 self.search('', return_matches=False) 408 409 def refresh_ids(self, ids): 410 self.cache.clear_caches(book_ids=ids) 411 try: 412 return list(map(self.id_to_index, ids)) 413 except ValueError: 414 pass 415 return None 416 417 def remove(self, book_id): 418 try: 419 self._map = tuple(bid for bid in self._map if bid != book_id) 420 except ValueError: 421 pass 422 try: 423 self._map_filtered = tuple(bid for bid in self._map_filtered if bid != book_id) 424 except ValueError: 425 pass 426 427 def books_deleted(self, ids): 428 for book_id in ids: 429 self.remove(book_id) 430 431 def books_added(self, ids): 432 ids = tuple(ids) 433 self._map = ids + self._map 434 self._map_filtered = ids + self._map_filtered 435 if prefs['mark_new_books']: 436 self.toggle_marked_ids(ids) 437