1#!/usr/local/bin/python3.8 2# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai 3 4 5__license__ = 'GPL v3' 6__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' 7__docformat__ = 'restructuredtext en' 8 9import operator 10import os 11import random 12import shutil 13import sys 14import traceback 15from collections import defaultdict 16from collections.abc import MutableSet, Set 17from functools import partial, wraps 18from io import BytesIO 19from threading import Lock 20from time import time 21 22from calibre import as_unicode, isbytestring 23from calibre.constants import iswindows, preferred_encoding 24from calibre.customize.ui import ( 25 run_plugins_on_import, run_plugins_on_postadd, run_plugins_on_postimport 26) 27from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list 28from calibre.db.annotations import merge_annotations 29from calibre.db.categories import get_categories 30from calibre.db.errors import NoSuchBook, NoSuchFormat 31from calibre.db.fields import IDENTITY, InvalidLinkTable, create_field 32from calibre.db.lazy import FormatMetadata, FormatsList, ProxyMetadata 33from calibre.db.listeners import EventDispatcher, EventType 34from calibre.db.locking import ( 35 DowngradeLockError, LockingError, SafeReadLock, create_locks, try_lock 36) 37from calibre.db.search import Search 38from calibre.db.tables import VirtualTable 39from calibre.db.utils import type_safe_sort_key_function 40from calibre.db.write import get_series_values, uniq 41from calibre.ebooks import check_ebook_format 42from calibre.ebooks.metadata import author_to_author_sort, string_to_authors 43from calibre.ebooks.metadata.book.base import Metadata 44from calibre.ebooks.metadata.opf2 import metadata_to_opf 45from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile, base_dir 46from calibre.utils.config import prefs, tweaks 47from calibre.utils.date import UNDEFINED_DATE, now as nowf, utcnow 48from calibre.utils.icu import sort_key 49from calibre.utils.localization import canonicalize_lang 50from polyglot.builtins import cmp, iteritems, itervalues, string_or_bytes 51 52 53def api(f): 54 f.is_cache_api = True 55 return f 56 57 58def read_api(f): 59 f = api(f) 60 f.is_read_api = True 61 return f 62 63 64def write_api(f): 65 f = api(f) 66 f.is_read_api = False 67 return f 68 69 70def wrap_simple(lock, func): 71 @wraps(func) 72 def call_func_with_lock(*args, **kwargs): 73 try: 74 with lock: 75 return func(*args, **kwargs) 76 except DowngradeLockError: 77 # We already have an exclusive lock, no need to acquire a shared 78 # lock. See the safe_read_lock properties' documentation for why 79 # this is necessary. 80 return func(*args, **kwargs) 81 return call_func_with_lock 82 83 84def run_import_plugins(path_or_stream, fmt): 85 fmt = fmt.lower() 86 if hasattr(path_or_stream, 'seek'): 87 path_or_stream.seek(0) 88 pt = PersistentTemporaryFile('_import_plugin.'+fmt) 89 shutil.copyfileobj(path_or_stream, pt, 1024**2) 90 pt.close() 91 path = pt.name 92 else: 93 path = path_or_stream 94 return run_plugins_on_import(path, fmt) 95 96 97def _add_newbook_tag(mi): 98 tags = prefs['new_book_tags'] 99 if tags: 100 for tag in [t.strip() for t in tags]: 101 if tag: 102 if not mi.tags: 103 mi.tags = [tag] 104 elif tag not in mi.tags: 105 mi.tags.append(tag) 106 107 108def _add_default_custom_column_values(mi, fm): 109 cols = fm.custom_field_metadata(include_composites=False) 110 for cc,col in iteritems(cols): 111 dv = col['display'].get('default_value', None) 112 try: 113 if dv is not None: 114 if not mi.get_user_metadata(cc, make_copy=False): 115 mi.set_user_metadata(cc, col) 116 dt = col['datatype'] 117 if dt == 'datetime' and icu_lower(dv) == 'now': 118 dv = nowf() 119 mi.set(cc, dv) 120 except: 121 traceback.print_exc() 122 123 124dynamic_category_preferences = frozenset({'grouped_search_make_user_categories', 'grouped_search_terms', 'user_categories'}) 125 126 127class Cache: 128 129 ''' 130 An in-memory cache of the metadata.db file from a calibre library. 131 This class also serves as a threadsafe API for accessing the database. 132 The in-memory cache is maintained in normal form for maximum performance. 133 134 SQLITE is simply used as a way to read and write from metadata.db robustly. 135 All table reading/sorting/searching/caching logic is re-implemented. This 136 was necessary for maximum performance and flexibility. 137 ''' 138 EventType = EventType 139 140 def __init__(self, backend): 141 self.backend = backend 142 self.event_dispatcher = EventDispatcher() 143 self.fields = {} 144 self.composites = {} 145 self.read_lock, self.write_lock = create_locks() 146 self.format_metadata_cache = defaultdict(dict) 147 self.formatter_template_cache = {} 148 self.dirtied_cache = {} 149 self.vls_for_books_cache = None 150 self.vls_cache_lock = Lock() 151 self.dirtied_sequence = 0 152 self.cover_caches = set() 153 self.clear_search_cache_count = 0 154 155 # Implement locking for all simple read/write API methods 156 # An unlocked version of the method is stored with the name starting 157 # with a leading underscore. Use the unlocked versions when the lock 158 # has already been acquired. 159 for name in dir(self): 160 func = getattr(self, name) 161 ira = getattr(func, 'is_read_api', None) 162 if ira is not None: 163 # Save original function 164 setattr(self, '_'+name, func) 165 # Wrap it in a lock 166 lock = self.read_lock if ira else self.write_lock 167 setattr(self, name, wrap_simple(lock, func)) 168 169 self._search_api = Search(self, 'saved_searches', self.field_metadata.get_search_terms()) 170 self.initialize_dynamic() 171 172 @property 173 def new_api(self): 174 return self 175 176 @property 177 def library_id(self): 178 return self.backend.library_id 179 180 @property 181 def dbpath(self): 182 return self.backend.dbpath 183 184 @property 185 def safe_read_lock(self): 186 ''' A safe read lock is a lock that does nothing if the thread already 187 has a write lock, otherwise it acquires a read lock. This is necessary 188 to prevent DowngradeLockErrors, which can happen when updating the 189 search cache in the presence of composite columns. Updating the search 190 cache holds an exclusive lock, but searching a composite column 191 involves reading field values via ProxyMetadata which tries to get a 192 shared lock. There may be other scenarios that trigger this as well. 193 194 This property returns a new lock object on every access. This lock 195 object is not recursive (for performance) and must only be used in a 196 with statement as ``with cache.safe_read_lock:`` otherwise bad things 197 will happen.''' 198 return SafeReadLock(self.read_lock) 199 200 @write_api 201 def ensure_has_search_category(self, fail_on_existing=True): 202 if len(self._search_api.saved_searches.names()) > 0: 203 self.field_metadata.add_search_category(label='search', name=_('Saved searches'), fail_on_existing=fail_on_existing) 204 205 def _initialize_dynamic_categories(self): 206 # Reconstruct the user categories, putting them into field_metadata 207 fm = self.field_metadata 208 fm.remove_dynamic_categories() 209 for user_cat in sorted(self._pref('user_categories', {}), key=sort_key): 210 cat_name = '@' + user_cat # add the '@' to avoid name collision 211 while cat_name: 212 try: 213 fm.add_user_category(label=cat_name, name=user_cat) 214 except ValueError: 215 break # Can happen since we are removing dots and adding parent categories ourselves 216 cat_name = cat_name.rpartition('.')[0] 217 218 # add grouped search term user categories 219 muc = frozenset(self._pref('grouped_search_make_user_categories', [])) 220 for cat in sorted(self._pref('grouped_search_terms', {}), key=sort_key): 221 if cat in muc: 222 # There is a chance that these can be duplicates of an existing 223 # user category. Print the exception and continue. 224 try: 225 self.field_metadata.add_user_category(label='@' + cat, name=cat) 226 except ValueError: 227 traceback.print_exc() 228 self._ensure_has_search_category() 229 230 self.field_metadata.add_grouped_search_terms( 231 self._pref('grouped_search_terms', {})) 232 self._refresh_search_locations() 233 234 @write_api 235 def initialize_dynamic(self): 236 self.backend.dirty_books_with_dirtied_annotations() 237 self.dirtied_cache = {x:i for i, x in enumerate(self.backend.dirtied_books())} 238 if self.dirtied_cache: 239 self.dirtied_sequence = max(itervalues(self.dirtied_cache))+1 240 self._initialize_dynamic_categories() 241 242 @write_api 243 def initialize_template_cache(self): 244 self.formatter_template_cache = {} 245 246 @write_api 247 def set_user_template_functions(self, user_template_functions): 248 self.backend.set_user_template_functions(user_template_functions) 249 250 @write_api 251 def clear_composite_caches(self, book_ids=None): 252 for field in itervalues(self.composites): 253 field.clear_caches(book_ids=book_ids) 254 255 @write_api 256 def clear_search_caches(self, book_ids=None): 257 self.clear_search_cache_count += 1 258 self._search_api.update_or_clear(self, book_ids) 259 self.vls_for_books_cache = None 260 261 @read_api 262 def last_modified(self): 263 return self.backend.last_modified() 264 265 @write_api 266 def clear_caches(self, book_ids=None, template_cache=True, search_cache=True): 267 if template_cache: 268 self._initialize_template_cache() # Clear the formatter template cache 269 for field in itervalues(self.fields): 270 if hasattr(field, 'clear_caches'): 271 field.clear_caches(book_ids=book_ids) # Clear the composite cache and ondevice caches 272 if book_ids: 273 for book_id in book_ids: 274 self.format_metadata_cache.pop(book_id, None) 275 else: 276 self.format_metadata_cache.clear() 277 if search_cache: 278 self._clear_search_caches(book_ids) 279 280 @write_api 281 def reload_from_db(self, clear_caches=True): 282 if clear_caches: 283 self._clear_caches() 284 with self.backend.conn: # Prevent other processes, such as calibredb from interrupting the reload by locking the db 285 self.backend.prefs.load_from_db() 286 self._search_api.saved_searches.load_from_db() 287 for field in itervalues(self.fields): 288 if hasattr(field, 'table'): 289 field.table.read(self.backend) # Reread data from metadata.db 290 291 @property 292 def field_metadata(self): 293 return self.backend.field_metadata 294 295 def _get_metadata(self, book_id, get_user_categories=True): # {{{ 296 mi = Metadata(None, template_cache=self.formatter_template_cache) 297 298 mi._proxy_metadata = ProxyMetadata(self, book_id, formatter=mi.formatter) 299 300 author_ids = self._field_ids_for('authors', book_id) 301 adata = self._author_data(author_ids) 302 aut_list = [adata[i] for i in author_ids] 303 aum = [] 304 aus = {} 305 aul = {} 306 for rec in aut_list: 307 aut = rec['name'] 308 aum.append(aut) 309 aus[aut] = rec['sort'] 310 aul[aut] = rec['link'] 311 mi.title = self._field_for('title', book_id, 312 default_value=_('Unknown')) 313 mi.authors = aum 314 mi.author_sort = self._field_for('author_sort', book_id, 315 default_value=_('Unknown')) 316 mi.author_sort_map = aus 317 mi.author_link_map = aul 318 mi.comments = self._field_for('comments', book_id) 319 mi.publisher = self._field_for('publisher', book_id) 320 n = utcnow() 321 mi.timestamp = self._field_for('timestamp', book_id, default_value=n) 322 mi.pubdate = self._field_for('pubdate', book_id, default_value=n) 323 mi.uuid = self._field_for('uuid', book_id, 324 default_value='dummy') 325 mi.title_sort = self._field_for('sort', book_id, 326 default_value=_('Unknown')) 327 mi.last_modified = self._field_for('last_modified', book_id, 328 default_value=n) 329 formats = self._field_for('formats', book_id) 330 mi.format_metadata = {} 331 mi.languages = list(self._field_for('languages', book_id)) 332 if not formats: 333 good_formats = None 334 else: 335 mi.format_metadata = FormatMetadata(self, book_id, formats) 336 good_formats = FormatsList(sorted(formats), mi.format_metadata) 337 # These three attributes are returned by the db2 get_metadata(), 338 # however, we dont actually use them anywhere other than templates, so 339 # they have been removed, to avoid unnecessary overhead. The templates 340 # all use _proxy_metadata. 341 # mi.book_size = self._field_for('size', book_id, default_value=0) 342 # mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') 343 # mi.db_approx_formats = formats 344 mi.formats = good_formats 345 mi.has_cover = _('Yes') if self._field_for('cover', book_id, 346 default_value=False) else '' 347 mi.tags = list(self._field_for('tags', book_id, default_value=())) 348 mi.series = self._field_for('series', book_id) 349 if mi.series: 350 mi.series_index = self._field_for('series_index', book_id, 351 default_value=1.0) 352 mi.rating = self._field_for('rating', book_id) 353 mi.set_identifiers(self._field_for('identifiers', book_id, 354 default_value={})) 355 mi.application_id = book_id 356 mi.id = book_id 357 composites = [] 358 for key, meta in self.field_metadata.custom_iteritems(): 359 mi.set_user_metadata(key, meta) 360 if meta['datatype'] == 'composite': 361 composites.append(key) 362 else: 363 val = self._field_for(key, book_id) 364 if isinstance(val, tuple): 365 val = list(val) 366 extra = self._field_for(key+'_index', book_id) 367 mi.set(key, val=val, extra=extra) 368 for key in composites: 369 mi.set(key, val=self._composite_for(key, book_id, mi)) 370 371 user_cat_vals = {} 372 if get_user_categories: 373 user_cats = self._pref('user_categories', {}) 374 for ucat in user_cats: 375 res = [] 376 for name,cat,ign in user_cats[ucat]: 377 v = mi.get(cat, None) 378 if isinstance(v, list): 379 if name in v: 380 res.append([name,cat]) 381 elif name == v: 382 res.append([name,cat]) 383 user_cat_vals[ucat] = res 384 mi.user_categories = user_cat_vals 385 386 return mi 387 # }}} 388 389 @api 390 def init(self): 391 ''' 392 Initialize this cache with data from the backend. 393 ''' 394 with self.write_lock: 395 self.backend.read_tables() 396 bools_are_tristate = self.backend.prefs['bools_are_tristate'] 397 398 for field, table in iteritems(self.backend.tables): 399 self.fields[field] = create_field(field, table, bools_are_tristate, 400 self.backend.get_template_functions) 401 if table.metadata['datatype'] == 'composite': 402 self.composites[field] = self.fields[field] 403 404 self.fields['ondevice'] = create_field('ondevice', 405 VirtualTable('ondevice'), bools_are_tristate, 406 self.backend.get_template_functions) 407 408 for name, field in iteritems(self.fields): 409 if name[0] == '#' and name.endswith('_index'): 410 field.series_field = self.fields[name[:-len('_index')]] 411 self.fields[name[:-len('_index')]].index_field = field 412 elif name == 'series_index': 413 field.series_field = self.fields['series'] 414 self.fields['series'].index_field = field 415 elif name == 'authors': 416 field.author_sort_field = self.fields['author_sort'] 417 elif name == 'title': 418 field.title_sort_field = self.fields['sort'] 419 if self.backend.prefs['update_all_last_mod_dates_on_start']: 420 self.update_last_modified(self.all_book_ids()) 421 self.backend.prefs.set('update_all_last_mod_dates_on_start', False) 422 423 # Cache Layer API {{{ 424 425 @write_api 426 def add_listener(self, event_callback_function): 427 ''' 428 Register a callback function that will be called after certain actions are 429 taken on this database. The function must take three arguments: 430 (:class:`EventType`, library_id, event_type_specific_data) 431 ''' 432 self.event_dispatcher.library_id = getattr(self, 'server_library_id', self.library_id) 433 self.event_dispatcher.add_listener(event_callback_function) 434 435 @write_api 436 def remove_listener(self, event_callback_function): 437 self.event_dispatcher.remove_listener(event_callback_function) 438 439 @read_api 440 def field_for(self, name, book_id, default_value=None): 441 ''' 442 Return the value of the field ``name`` for the book identified 443 by ``book_id``. If no such book exists or it has no defined 444 value for the field ``name`` or no such field exists, then 445 ``default_value`` is returned. 446 447 ``default_value`` is not used for title, title_sort, authors, author_sort 448 and series_index. This is because these always have values in the db. 449 ``default_value`` is used for all custom columns. 450 451 The returned value for is_multiple fields are always tuples, even when 452 no values are found (in other words, default_value is ignored). The 453 exception is identifiers for which the returned value is always a dict. 454 The returned tuples are always in link order, that is, the order in 455 which they were created. 456 ''' 457 if self.composites and name in self.composites: 458 return self.composite_for(name, book_id, 459 default_value=default_value) 460 try: 461 field = self.fields[name] 462 except KeyError: 463 return default_value 464 if field.is_multiple: 465 default_value = field.default_value 466 try: 467 return field.for_book(book_id, default_value=default_value) 468 except (KeyError, IndexError): 469 return default_value 470 471 @read_api 472 def fast_field_for(self, field_obj, book_id, default_value=None): 473 ' Same as field_for, except that it avoids the extra lookup to get the field object ' 474 if field_obj.is_composite: 475 return field_obj.get_value_with_cache(book_id, self._get_proxy_metadata) 476 if field_obj.is_multiple: 477 default_value = field_obj.default_value 478 try: 479 return field_obj.for_book(book_id, default_value=default_value) 480 except (KeyError, IndexError): 481 return default_value 482 483 @read_api 484 def all_field_for(self, field, book_ids, default_value=None): 485 ' Same as field_for, except that it operates on multiple books at once ' 486 field_obj = self.fields[field] 487 return {book_id:self._fast_field_for(field_obj, book_id, default_value=default_value) for book_id in book_ids} 488 489 @read_api 490 def composite_for(self, name, book_id, mi=None, default_value=''): 491 try: 492 f = self.fields[name] 493 except KeyError: 494 return default_value 495 496 if mi is None: 497 return f.get_value_with_cache(book_id, self._get_proxy_metadata) 498 else: 499 return f._render_composite_with_cache(book_id, mi, mi.formatter, mi.template_cache) 500 501 @read_api 502 def field_ids_for(self, name, book_id): 503 ''' 504 Return the ids (as a tuple) for the values that the field ``name`` has on the book 505 identified by ``book_id``. If there are no values, or no such book, or 506 no such field, an empty tuple is returned. 507 ''' 508 try: 509 return self.fields[name].ids_for_book(book_id) 510 except (KeyError, IndexError): 511 return () 512 513 @read_api 514 def books_for_field(self, name, item_id): 515 ''' 516 Return all the books associated with the item identified by 517 ``item_id``, where the item belongs to the field ``name``. 518 519 Returned value is a set of book ids, or the empty set if the item 520 or the field does not exist. 521 ''' 522 try: 523 return self.fields[name].books_for(item_id) 524 except (KeyError, IndexError): 525 return set() 526 527 @read_api 528 def all_book_ids(self, type=frozenset): 529 ''' 530 Frozen set of all known book ids. 531 ''' 532 return type(self.fields['uuid'].table.book_col_map) 533 534 @read_api 535 def all_field_ids(self, name): 536 ''' 537 Frozen set of ids for all values in the field ``name``. 538 ''' 539 return frozenset(iter(self.fields[name])) 540 541 @read_api 542 def all_field_names(self, field): 543 ''' Frozen set of all fields names (should only be used for many-one and many-many fields) ''' 544 if field == 'formats': 545 return frozenset(self.fields[field].table.col_book_map) 546 547 try: 548 return frozenset(self.fields[field].table.id_map.values()) 549 except AttributeError: 550 raise ValueError('%s is not a many-one or many-many field' % field) 551 552 @read_api 553 def get_usage_count_by_id(self, field): 554 ''' Return a mapping of id to usage count for all values of the specified 555 field, which must be a many-one or many-many field. ''' 556 try: 557 return {k:len(v) for k, v in iteritems(self.fields[field].table.col_book_map)} 558 except AttributeError: 559 raise ValueError('%s is not a many-one or many-many field' % field) 560 561 @read_api 562 def get_id_map(self, field): 563 ''' Return a mapping of id numbers to values for the specified field. 564 The field must be a many-one or many-many field, otherwise a ValueError 565 is raised. ''' 566 try: 567 return self.fields[field].table.id_map.copy() 568 except AttributeError: 569 if field == 'title': 570 return self.fields[field].table.book_col_map.copy() 571 raise ValueError('%s is not a many-one or many-many field' % field) 572 573 @read_api 574 def get_item_name(self, field, item_id): 575 ''' Return the item name for the item specified by item_id in the 576 specified field. See also :meth:`get_id_map`.''' 577 return self.fields[field].table.id_map[item_id] 578 579 @read_api 580 def get_item_id(self, field, item_name): 581 ' Return the item id for item_name (case-insensitive) ' 582 rmap = {icu_lower(v) if isinstance(v, str) else v:k for k, v in iteritems(self.fields[field].table.id_map)} 583 return rmap.get(icu_lower(item_name) if isinstance(item_name, str) else item_name, None) 584 585 @read_api 586 def get_item_ids(self, field, item_names): 587 ' Return the item id for item_name (case-insensitive) ' 588 rmap = {icu_lower(v) if isinstance(v, str) else v:k for k, v in iteritems(self.fields[field].table.id_map)} 589 return {name:rmap.get(icu_lower(name) if isinstance(name, str) else name, None) for name in item_names} 590 591 @read_api 592 def author_data(self, author_ids=None): 593 ''' 594 Return author data as a dictionary with keys: name, sort, link 595 596 If no authors with the specified ids are found an empty dictionary is 597 returned. If author_ids is None, data for all authors is returned. 598 ''' 599 af = self.fields['authors'] 600 if author_ids is None: 601 return {aid:af.author_data(aid) for aid in af.table.id_map} 602 return {aid:af.author_data(aid) for aid in author_ids if aid in af.table.id_map} 603 604 @read_api 605 def format_hash(self, book_id, fmt): 606 ''' Return the hash of the specified format for the specified book. The 607 kind of hash is backend dependent, but is usually SHA-256. ''' 608 try: 609 name = self.fields['formats'].format_fname(book_id, fmt) 610 path = self._field_for('path', book_id).replace('/', os.sep) 611 except: 612 raise NoSuchFormat('Record %d has no fmt: %s'%(book_id, fmt)) 613 return self.backend.format_hash(book_id, fmt, name, path) 614 615 @api 616 def format_metadata(self, book_id, fmt, allow_cache=True, update_db=False): 617 ''' 618 Return the path, size and mtime for the specified format for the specified book. 619 You should not use path unless you absolutely have to, 620 since accessing it directly breaks the threadsafe guarantees of this API. Instead use 621 the :meth:`copy_format_to` method. 622 623 :param allow_cache: If ``True`` cached values are used, otherwise a 624 slow filesystem access is done. The cache values could be out of date 625 if access was performed to the filesystem outside of this API. 626 627 :param update_db: If ``True`` The max_size field of the database is updated for this book. 628 ''' 629 if not fmt: 630 return {} 631 fmt = fmt.upper() 632 # allow_cache and update_db are mutually exclusive. Give priority to update_db 633 if allow_cache and not update_db: 634 x = self.format_metadata_cache[book_id].get(fmt, None) 635 if x is not None: 636 return x 637 with self.safe_read_lock: 638 try: 639 name = self.fields['formats'].format_fname(book_id, fmt) 640 path = self._field_for('path', book_id).replace('/', os.sep) 641 except: 642 return {} 643 644 ans = {} 645 if path and name: 646 ans = self.backend.format_metadata(book_id, fmt, name, path) 647 self.format_metadata_cache[book_id][fmt] = ans 648 if update_db and 'size' in ans: 649 with self.write_lock: 650 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, name, ans['size'], self.backend) 651 self.fields['size'].table.update_sizes({book_id: max_size}) 652 653 return ans 654 655 @read_api 656 def format_files(self, book_id): 657 field = self.fields['formats'] 658 fmts = field.table.book_col_map.get(book_id, ()) 659 return {fmt:field.format_fname(book_id, fmt) for fmt in fmts} 660 661 @read_api 662 def format_db_size(self, book_id, fmt): 663 field = self.fields['formats'] 664 return field.format_size(book_id, fmt) 665 666 @read_api 667 def pref(self, name, default=None, namespace=None): 668 ' Return the value for the specified preference or the value specified as ``default`` if the preference is not set. ' 669 if namespace is not None: 670 return self.backend.prefs.get_namespaced(namespace, name, default) 671 return self.backend.prefs.get(name, default) 672 673 @write_api 674 def set_pref(self, name, val, namespace=None): 675 ' Set the specified preference to the specified value. See also :meth:`pref`. ' 676 if namespace is not None: 677 self.backend.prefs.set_namespaced(namespace, name, val) 678 return 679 self.backend.prefs.set(name, val) 680 if name in ('grouped_search_terms', 'virtual_libraries'): 681 self._clear_search_caches() 682 if name in dynamic_category_preferences: 683 self._initialize_dynamic_categories() 684 685 @api 686 def get_metadata(self, book_id, 687 get_cover=False, get_user_categories=True, cover_as_data=False): 688 ''' 689 Return metadata for the book identified by book_id as a :class:`calibre.ebooks.metadata.book.base.Metadata` object. 690 Note that the list of formats is not verified. If get_cover is True, 691 the cover is returned, either a path to temp file as mi.cover or if 692 cover_as_data is True then as mi.cover_data. 693 ''' 694 695 # Check if virtual_libraries_for_books rebuilt its cache. If it did then 696 # we must clear the composite caches so the new data can be taken into 697 # account. Clearing the caches requires getting a write lock, so it must 698 # be done outside of the closure of _get_metadata(). 699 composite_cache_needs_to_be_cleared = False 700 with self.safe_read_lock: 701 vl_cache_was_none = self.vls_for_books_cache is None 702 mi = self._get_metadata(book_id, get_user_categories=get_user_categories) 703 if vl_cache_was_none and self.vls_for_books_cache is not None: 704 composite_cache_needs_to_be_cleared = True 705 if composite_cache_needs_to_be_cleared: 706 try: 707 self.clear_composite_caches() 708 except LockingError: 709 # We can't clear the composite caches because a read lock is set. 710 # As a consequence the value of a composite column that calls 711 # virtual_libraries() might be wrong. Oh well. Log and keep running. 712 print("Couldn't get write lock after vls_for_books_cache was loaded", file=sys.stderr) 713 traceback.print_exc() 714 715 if get_cover: 716 if cover_as_data: 717 cdata = self.cover(book_id) 718 if cdata: 719 mi.cover_data = ('jpeg', cdata) 720 else: 721 mi.cover = self.cover(book_id, as_path=True) 722 723 return mi 724 725 @read_api 726 def get_proxy_metadata(self, book_id): 727 ''' Like :meth:`get_metadata` except that it returns a ProxyMetadata 728 object that only reads values from the database on demand. This is much 729 faster than get_metadata when only a small number of fields need to be 730 accessed from the returned metadata object. ''' 731 return ProxyMetadata(self, book_id) 732 733 @api 734 def cover(self, book_id, 735 as_file=False, as_image=False, as_path=False): 736 ''' 737 Return the cover image or None. By default, returns the cover as a 738 bytestring. 739 740 WARNING: Using as_path will copy the cover to a temp file and return 741 the path to the temp file. You should delete the temp file when you are 742 done with it. 743 744 :param as_file: If True return the image as an open file object (a SpooledTemporaryFile) 745 :param as_image: If True return the image as a QImage object 746 :param as_path: If True return the image as a path pointing to a 747 temporary file 748 ''' 749 if as_file: 750 ret = SpooledTemporaryFile(SPOOL_SIZE) 751 if not self.copy_cover_to(book_id, ret): 752 return 753 ret.seek(0) 754 elif as_path: 755 pt = PersistentTemporaryFile('_dbcover.jpg') 756 with pt: 757 if not self.copy_cover_to(book_id, pt): 758 return 759 ret = pt.name 760 else: 761 buf = BytesIO() 762 if not self.copy_cover_to(book_id, buf): 763 return 764 ret = buf.getvalue() 765 if as_image: 766 from qt.core import QImage 767 i = QImage() 768 i.loadFromData(ret) 769 ret = i 770 return ret 771 772 @read_api 773 def cover_or_cache(self, book_id, timestamp): 774 try: 775 path = self._field_for('path', book_id).replace('/', os.sep) 776 except AttributeError: 777 return False, None, None 778 return self.backend.cover_or_cache(path, timestamp) 779 780 @read_api 781 def cover_last_modified(self, book_id): 782 try: 783 path = self._field_for('path', book_id).replace('/', os.sep) 784 except AttributeError: 785 return 786 return self.backend.cover_last_modified(path) 787 788 @read_api 789 def copy_cover_to(self, book_id, dest, use_hardlink=False, report_file_size=None): 790 ''' 791 Copy the cover to the file like object ``dest``. Returns False 792 if no cover exists or dest is the same file as the current cover. 793 dest can also be a path in which case the cover is 794 copied to it if and only if the path is different from the current path (taking 795 case sensitivity into account). 796 ''' 797 try: 798 path = self._field_for('path', book_id).replace('/', os.sep) 799 except AttributeError: 800 return False 801 802 return self.backend.copy_cover_to(path, dest, use_hardlink=use_hardlink, 803 report_file_size=report_file_size) 804 805 @write_api 806 def compress_covers(self, book_ids, jpeg_quality=100, progress_callback=None): 807 ''' 808 Compress the cover images for the specified books. A compression quality of 100 809 will perform lossless compression, otherwise lossy compression. 810 811 The progress callback will be called with the book_id and the old and new sizes 812 for each book that has been processed. If an error occurs, the new size will 813 be a string with the error details. 814 ''' 815 jpeg_quality = max(10, min(jpeg_quality, 100)) 816 path_map = {} 817 for book_id in book_ids: 818 try: 819 path_map[book_id] = self._field_for('path', book_id).replace('/', os.sep) 820 except AttributeError: 821 continue 822 self.backend.compress_covers(path_map, jpeg_quality, progress_callback) 823 824 @read_api 825 def copy_format_to(self, book_id, fmt, dest, use_hardlink=False, report_file_size=None): 826 ''' 827 Copy the format ``fmt`` to the file like object ``dest``. If the 828 specified format does not exist, raises :class:`NoSuchFormat` error. 829 dest can also be a path (to a file), in which case the format is copied to it, iff 830 the path is different from the current path (taking case sensitivity 831 into account). 832 ''' 833 fmt = (fmt or '').upper() 834 try: 835 name = self.fields['formats'].format_fname(book_id, fmt) 836 path = self._field_for('path', book_id).replace('/', os.sep) 837 except (KeyError, AttributeError): 838 raise NoSuchFormat('Record %d has no %s file'%(book_id, fmt)) 839 840 return self.backend.copy_format_to(book_id, fmt, name, path, dest, 841 use_hardlink=use_hardlink, report_file_size=report_file_size) 842 843 @read_api 844 def format_abspath(self, book_id, fmt): 845 ''' 846 Return absolute path to the e-book file of format `format`. You should 847 almost never use this, as it breaks the threadsafe promise of this API. 848 Instead use, :meth:`copy_format_to`. 849 850 Currently used only in calibredb list, the viewer, edit book, 851 compare_format to original format, open with, bulk metadata edit and 852 the catalogs (via get_data_as_dict()). 853 854 Apart from the viewer, open with and edit book, I don't believe any of 855 the others do any file write I/O with the results of this call. 856 ''' 857 fmt = (fmt or '').upper() 858 try: 859 path = self._field_for('path', book_id).replace('/', os.sep) 860 except: 861 return None 862 if path: 863 if fmt == '__COVER_INTERNAL__': 864 return self.backend.cover_abspath(book_id, path) 865 else: 866 try: 867 name = self.fields['formats'].format_fname(book_id, fmt) 868 except: 869 return None 870 if name: 871 return self.backend.format_abspath(book_id, fmt, name, path) 872 873 @read_api 874 def has_format(self, book_id, fmt): 875 'Return True iff the format exists on disk' 876 fmt = (fmt or '').upper() 877 try: 878 name = self.fields['formats'].format_fname(book_id, fmt) 879 path = self._field_for('path', book_id).replace('/', os.sep) 880 except: 881 return False 882 return self.backend.has_format(book_id, fmt, name, path) 883 884 @api 885 def save_original_format(self, book_id, fmt): 886 ' Save a copy of the specified format as ORIGINAL_FORMAT, overwriting any existing ORIGINAL_FORMAT. ' 887 fmt = fmt.upper() 888 if 'ORIGINAL' in fmt: 889 raise ValueError('Cannot save original of an original fmt') 890 fmtfile = self.format(book_id, fmt, as_file=True) 891 if fmtfile is None: 892 return False 893 with fmtfile: 894 nfmt = 'ORIGINAL_'+fmt 895 return self.add_format(book_id, nfmt, fmtfile, run_hooks=False) 896 897 @write_api 898 def restore_original_format(self, book_id, original_fmt): 899 ''' Restore the specified format from the previously saved 900 ORIGINAL_FORMAT, if any. Return True on success. The ORIGINAL_FORMAT is 901 deleted after a successful restore. ''' 902 original_fmt = original_fmt.upper() 903 fmt = original_fmt.partition('_')[2] 904 try: 905 ofmt_name = self.fields['formats'].format_fname(book_id, original_fmt) 906 path = self._field_for('path', book_id).replace('/', os.sep) 907 except Exception: 908 return False 909 if self.backend.is_format_accessible(book_id, original_fmt, ofmt_name, path): 910 self.add_format(book_id, fmt, BytesIO(), run_hooks=False) 911 fmt_name = self.fields['formats'].format_fname(book_id, fmt) 912 file_size = self.backend.rename_format_file(book_id, ofmt_name, original_fmt, fmt_name, fmt, path) 913 self.fields['formats'].table.update_fmt(book_id, fmt, fmt_name, file_size, self.backend) 914 self._remove_formats({book_id:(original_fmt,)}) 915 return True 916 return False 917 918 @read_api 919 def formats(self, book_id, verify_formats=True): 920 ''' 921 Return tuple of all formats for the specified book. If verify_formats 922 is True, verifies that the files exist on disk. 923 ''' 924 ans = self.field_for('formats', book_id) 925 if verify_formats and ans: 926 try: 927 path = self._field_for('path', book_id).replace('/', os.sep) 928 except: 929 return () 930 931 def verify(fmt): 932 try: 933 name = self.fields['formats'].format_fname(book_id, fmt) 934 except: 935 return False 936 return self.backend.has_format(book_id, fmt, name, path) 937 938 ans = tuple(x for x in ans if verify(x)) 939 return ans 940 941 @api 942 def format(self, book_id, fmt, as_file=False, as_path=False, preserve_filename=False): 943 ''' 944 Return the e-book format as a bytestring or `None` if the format doesn't exist, 945 or we don't have permission to write to the e-book file. 946 947 :param as_file: If True the e-book format is returned as a file object. Note 948 that the file object is a SpooledTemporaryFile, so if what you want to 949 do is copy the format to another file, use :meth:`copy_format_to` 950 instead for performance. 951 :param as_path: Copies the format file to a temp file and returns the 952 path to the temp file 953 :param preserve_filename: If True and returning a path the filename is 954 the same as that used in the library. Note that using 955 this means that repeated calls yield the same 956 temp file (which is re-created each time) 957 ''' 958 fmt = (fmt or '').upper() 959 ext = ('.'+fmt.lower()) if fmt else '' 960 if as_path: 961 if preserve_filename: 962 with self.safe_read_lock: 963 try: 964 fname = self.fields['formats'].format_fname(book_id, fmt) 965 except: 966 return None 967 fname += ext 968 969 bd = base_dir() 970 d = os.path.join(bd, 'format_abspath') 971 try: 972 os.makedirs(d) 973 except: 974 pass 975 ret = os.path.join(d, fname) 976 try: 977 self.copy_format_to(book_id, fmt, ret) 978 except NoSuchFormat: 979 return None 980 else: 981 with PersistentTemporaryFile(ext) as pt: 982 try: 983 self.copy_format_to(book_id, fmt, pt) 984 except NoSuchFormat: 985 return None 986 ret = pt.name 987 elif as_file: 988 with self.safe_read_lock: 989 try: 990 fname = self.fields['formats'].format_fname(book_id, fmt) 991 except: 992 return None 993 fname += ext 994 995 ret = SpooledTemporaryFile(SPOOL_SIZE) 996 try: 997 self.copy_format_to(book_id, fmt, ret) 998 except NoSuchFormat: 999 return None 1000 ret.seek(0) 1001 # Various bits of code try to use the name as the default 1002 # title when reading metadata, so set it 1003 ret.name = fname 1004 else: 1005 buf = BytesIO() 1006 try: 1007 self.copy_format_to(book_id, fmt, buf) 1008 except NoSuchFormat: 1009 return None 1010 1011 ret = buf.getvalue() 1012 1013 return ret 1014 1015 @read_api 1016 def multisort(self, fields, ids_to_sort=None, virtual_fields=None): 1017 ''' 1018 Return a list of sorted book ids. If ids_to_sort is None, all book ids 1019 are returned. 1020 1021 fields must be a list of 2-tuples of the form (field_name, 1022 ascending=True or False). The most significant field is the first 1023 2-tuple. 1024 ''' 1025 ids_to_sort = self._all_book_ids() if ids_to_sort is None else ids_to_sort 1026 get_metadata = self._get_proxy_metadata 1027 lang_map = self.fields['languages'].book_value_map 1028 virtual_fields = virtual_fields or {} 1029 1030 fm = {'title':'sort', 'authors':'author_sort'} 1031 1032 def sort_key_func(field): 1033 'Handle series type fields, virtual fields and the id field' 1034 idx = field + '_index' 1035 is_series = idx in self.fields 1036 try: 1037 func = self.fields[fm.get(field, field)].sort_keys_for_books(get_metadata, lang_map) 1038 except KeyError: 1039 if field == 'id': 1040 return IDENTITY 1041 else: 1042 return virtual_fields[fm.get(field, field)].sort_keys_for_books(get_metadata, lang_map) 1043 if is_series: 1044 idx_func = self.fields[idx].sort_keys_for_books(get_metadata, lang_map) 1045 1046 def skf(book_id): 1047 return (func(book_id), idx_func(book_id)) 1048 return skf 1049 return func 1050 1051 # Sort only once on any given field 1052 fields = uniq(fields, operator.itemgetter(0)) 1053 1054 if len(fields) == 1: 1055 keyfunc = sort_key_func(fields[0][0]) 1056 reverse = not fields[0][1] 1057 try: 1058 return sorted(ids_to_sort, key=keyfunc, reverse=reverse) 1059 except Exception as err: 1060 print('Failed to sort database on field:', fields[0][0], 'with error:', err, file=sys.stderr) 1061 try: 1062 return sorted(ids_to_sort, key=type_safe_sort_key_function(keyfunc), reverse=reverse) 1063 except Exception as err: 1064 print('Failed to type-safe sort database on field:', fields[0][0], 'with error:', err, file=sys.stderr) 1065 return sorted(ids_to_sort, reverse=reverse) 1066 sort_key_funcs = tuple(sort_key_func(field) for field, order in fields) 1067 orders = tuple(1 if order else -1 for _, order in fields) 1068 Lazy = object() # Lazy load the sort keys for sub-sort fields 1069 1070 class SortKey: 1071 1072 __slots__ = 'book_id', 'sort_key' 1073 1074 def __init__(self, book_id): 1075 self.book_id = book_id 1076 # Calculate only the first sub-sort key since that will always be used 1077 self.sort_key = [key(book_id) if i == 0 else Lazy for i, key in enumerate(sort_key_funcs)] 1078 1079 def compare_to_other(self, other): 1080 for i, (order, self_key, other_key) in enumerate(zip(orders, self.sort_key, other.sort_key)): 1081 if self_key is Lazy: 1082 self_key = self.sort_key[i] = sort_key_funcs[i](self.book_id) 1083 if other_key is Lazy: 1084 other_key = other.sort_key[i] = sort_key_funcs[i](other.book_id) 1085 ans = cmp(self_key, other_key) 1086 if ans != 0: 1087 return ans * order 1088 return 0 1089 1090 def __eq__(self, other): 1091 return self.compare_to_other(other) == 0 1092 1093 def __ne__(self, other): 1094 return self.compare_to_other(other) != 0 1095 1096 def __lt__(self, other): 1097 return self.compare_to_other(other) < 0 1098 1099 def __le__(self, other): 1100 return self.compare_to_other(other) <= 0 1101 1102 def __gt__(self, other): 1103 return self.compare_to_other(other) > 0 1104 1105 def __ge__(self, other): 1106 return self.compare_to_other(other) >= 0 1107 1108 return sorted(ids_to_sort, key=SortKey) 1109 1110 @read_api 1111 def search(self, query, restriction='', virtual_fields=None, book_ids=None): 1112 ''' 1113 Search the database for the specified query, returning a set of matched book ids. 1114 1115 :param restriction: A restriction that is ANDed to the specified query. Note that 1116 restrictions are cached, therefore the search for a AND b will be slower than a with restriction b. 1117 1118 :param virtual_fields: Used internally (virtual fields such as on_device to search over). 1119 1120 :param book_ids: If not None, a set of book ids for which books will 1121 be searched instead of searching all books. 1122 ''' 1123 return self._search_api(self, query, restriction, virtual_fields=virtual_fields, book_ids=book_ids) 1124 1125 @read_api 1126 def books_in_virtual_library(self, vl, search_restriction=None, virtual_fields=None): 1127 ' Return the set of books in the specified virtual library ' 1128 vl = self._pref('virtual_libraries', {}).get(vl) if vl else None 1129 if not vl and not search_restriction: 1130 return self.all_book_ids() 1131 # We utilize the search restriction cache to speed this up 1132 srch = partial(self._search, virtual_fields=virtual_fields) 1133 if vl: 1134 if search_restriction: 1135 return frozenset(srch('', vl) & srch('', search_restriction)) 1136 return frozenset(srch('', vl)) 1137 return frozenset(srch('', search_restriction)) 1138 1139 @read_api 1140 def number_of_books_in_virtual_library(self, vl=None, search_restriction=None): 1141 if not vl and not search_restriction: 1142 return len(self.fields['uuid'].table.book_col_map) 1143 return len(self.books_in_virtual_library(vl, search_restriction)) 1144 1145 @api 1146 def get_categories(self, sort='name', book_ids=None, already_fixed=None, 1147 first_letter_sort=False): 1148 ' Used internally to implement the Tag Browser ' 1149 try: 1150 with self.safe_read_lock: 1151 return get_categories(self, sort=sort, book_ids=book_ids, 1152 first_letter_sort=first_letter_sort) 1153 except InvalidLinkTable as err: 1154 bad_field = err.field_name 1155 if bad_field == already_fixed: 1156 raise 1157 with self.write_lock: 1158 self.fields[bad_field].table.fix_link_table(self.backend) 1159 return self.get_categories(sort=sort, book_ids=book_ids, already_fixed=bad_field) 1160 1161 @write_api 1162 def update_last_modified(self, book_ids, now=None): 1163 if book_ids: 1164 if now is None: 1165 now = nowf() 1166 f = self.fields['last_modified'] 1167 f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) 1168 if self.composites: 1169 self._clear_composite_caches(book_ids) 1170 self._clear_search_caches(book_ids) 1171 1172 @write_api 1173 def mark_as_dirty(self, book_ids): 1174 self._update_last_modified(book_ids) 1175 already_dirtied = set(self.dirtied_cache).intersection(book_ids) 1176 new_dirtied = book_ids - already_dirtied 1177 already_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(already_dirtied)} 1178 if already_dirtied: 1179 self.dirtied_sequence = max(itervalues(already_dirtied)) + 1 1180 self.dirtied_cache.update(already_dirtied) 1181 if new_dirtied: 1182 self.backend.dirty_books(new_dirtied) 1183 new_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(new_dirtied)} 1184 self.dirtied_sequence = max(itervalues(new_dirtied)) + 1 1185 self.dirtied_cache.update(new_dirtied) 1186 1187 @write_api 1188 def commit_dirty_cache(self): 1189 if self.dirtied_cache: 1190 self.backend.dirty_books(self.dirtied_cache) 1191 1192 @write_api 1193 def check_dirtied_annotations(self): 1194 if not self.backend.dirty_books_with_dirtied_annotations(): 1195 return 1196 book_ids = set(self.backend.dirtied_books()) 1197 new_dirtied = book_ids - set(self.dirtied_cache) 1198 if new_dirtied: 1199 new_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(new_dirtied)} 1200 self.dirtied_sequence = max(itervalues(new_dirtied)) + 1 1201 self.dirtied_cache.update(new_dirtied) 1202 1203 @write_api 1204 def set_field(self, name, book_id_to_val_map, allow_case_change=True, do_path_update=True): 1205 ''' 1206 Set the values of the field specified by ``name``. Returns the set of all book ids that were affected by the change. 1207 1208 :param book_id_to_val_map: Mapping of book_ids to values that should be applied. 1209 :param allow_case_change: If True, the case of many-one or many-many fields will be changed. 1210 For example, if a book has the tag ``tag1`` and you set the tag for another book to ``Tag1`` 1211 then the both books will have the tag ``Tag1`` if allow_case_change is True, otherwise they will 1212 both have the tag ``tag1``. 1213 :param do_path_update: Used internally, you should never change it. 1214 ''' 1215 f = self.fields[name] 1216 is_series = f.metadata['datatype'] == 'series' 1217 update_path = name in {'title', 'authors'} 1218 if update_path and iswindows: 1219 paths = (x for x in (self._field_for('path', book_id) for book_id in book_id_to_val_map) if x) 1220 self.backend.windows_check_if_files_in_use(paths) 1221 1222 if is_series: 1223 bimap, simap = {}, {} 1224 sfield = self.fields[name + '_index'] 1225 for k, v in iteritems(book_id_to_val_map): 1226 if isinstance(v, string_or_bytes): 1227 v, sid = get_series_values(v) 1228 else: 1229 v = sid = None 1230 if sid is None and name.startswith('#'): 1231 sid = self._fast_field_for(sfield, k) 1232 sid = 1.0 if sid is None else sid # The value to be set the db link table 1233 bimap[k] = v 1234 if sid is not None: 1235 simap[k] = sid 1236 book_id_to_val_map = bimap 1237 1238 dirtied = f.writer.set_books( 1239 book_id_to_val_map, self.backend, allow_case_change=allow_case_change) 1240 1241 if is_series and simap: 1242 sf = self.fields[f.name+'_index'] 1243 dirtied |= sf.writer.set_books(simap, self.backend, allow_case_change=False) 1244 1245 if dirtied: 1246 if update_path and do_path_update: 1247 self._update_path(dirtied, mark_as_dirtied=False) 1248 self._mark_as_dirty(dirtied) 1249 self.event_dispatcher(EventType.metadata_changed, name, dirtied) 1250 return dirtied 1251 1252 @write_api 1253 def update_path(self, book_ids, mark_as_dirtied=True): 1254 for book_id in book_ids: 1255 title = self._field_for('title', book_id, default_value=_('Unknown')) 1256 try: 1257 author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] 1258 except IndexError: 1259 author = _('Unknown') 1260 self.backend.update_path(book_id, title, author, self.fields['path'], self.fields['formats']) 1261 self.format_metadata_cache.pop(book_id, None) 1262 if mark_as_dirtied: 1263 self._mark_as_dirty(book_ids) 1264 1265 @read_api 1266 def get_a_dirtied_book(self): 1267 if self.dirtied_cache: 1268 return random.choice(tuple(self.dirtied_cache)) 1269 return None 1270 1271 @read_api 1272 def get_metadata_for_dump(self, book_id): 1273 mi = None 1274 # get the current sequence number for this book to pass back to the 1275 # backup thread. This will avoid double calls in the case where the 1276 # thread has not done the work between the put and the get_metadata 1277 sequence = self.dirtied_cache.get(book_id, None) 1278 if sequence is not None: 1279 try: 1280 # While a book is being created, the path is empty. Don't bother to 1281 # try to write the opf, because it will go to the wrong folder. 1282 if self._field_for('path', book_id): 1283 mi = self._get_metadata(book_id) 1284 # Always set cover to cover.jpg. Even if cover doesn't exist, 1285 # no harm done. This way no need to call dirtied when 1286 # cover is set/removed 1287 mi.cover = 'cover.jpg' 1288 mi.all_annotations = self._all_annotations_for_book(book_id) 1289 except: 1290 # This almost certainly means that the book has been deleted while 1291 # the backup operation sat in the queue. 1292 import traceback 1293 traceback.print_exc() 1294 return mi, sequence 1295 1296 @write_api 1297 def clear_dirtied(self, book_id, sequence): 1298 # Clear the dirtied indicator for the books. This is used when fetching 1299 # metadata, creating an OPF, and writing a file are separated into steps. 1300 # The last step is clearing the indicator 1301 dc_sequence = self.dirtied_cache.get(book_id, None) 1302 if dc_sequence is None or sequence is None or dc_sequence == sequence: 1303 self.backend.mark_book_as_clean(book_id) 1304 self.dirtied_cache.pop(book_id, None) 1305 1306 @write_api 1307 def write_backup(self, book_id, raw): 1308 try: 1309 path = self._field_for('path', book_id).replace('/', os.sep) 1310 except: 1311 return 1312 1313 self.backend.write_backup(path, raw) 1314 1315 @read_api 1316 def dirty_queue_length(self): 1317 return len(self.dirtied_cache) 1318 1319 @read_api 1320 def read_backup(self, book_id): 1321 ''' Return the OPF metadata backup for the book as a bytestring or None 1322 if no such backup exists. ''' 1323 try: 1324 path = self._field_for('path', book_id).replace('/', os.sep) 1325 except: 1326 return 1327 1328 try: 1329 return self.backend.read_backup(path) 1330 except OSError: 1331 return None 1332 1333 @write_api 1334 def dump_metadata(self, book_ids=None, remove_from_dirtied=True, 1335 callback=None): 1336 # Write metadata for each record to an individual OPF file. If callback 1337 # is not None, it is called once at the start with the number of book_ids 1338 # being processed. And once for every book_id, with arguments (book_id, 1339 # mi, ok). 1340 if book_ids is None: 1341 book_ids = set(self.dirtied_cache) 1342 1343 if callback is not None: 1344 callback(len(book_ids), True, False) 1345 1346 for book_id in book_ids: 1347 if self._field_for('path', book_id) is None: 1348 if callback is not None: 1349 callback(book_id, None, False) 1350 continue 1351 mi, sequence = self._get_metadata_for_dump(book_id) 1352 if mi is None: 1353 if callback is not None: 1354 callback(book_id, mi, False) 1355 continue 1356 try: 1357 raw = metadata_to_opf(mi) 1358 self._write_backup(book_id, raw) 1359 if remove_from_dirtied: 1360 self._clear_dirtied(book_id, sequence) 1361 except: 1362 pass 1363 if callback is not None: 1364 callback(book_id, mi, True) 1365 1366 @write_api 1367 def set_cover(self, book_id_data_map): 1368 ''' Set the cover for this book. The data can be either a QImage, 1369 QPixmap, file object or bytestring. It can also be None, in which 1370 case any existing cover is removed. ''' 1371 1372 for book_id, data in iteritems(book_id_data_map): 1373 try: 1374 path = self._field_for('path', book_id).replace('/', os.sep) 1375 except AttributeError: 1376 self._update_path((book_id,)) 1377 path = self._field_for('path', book_id).replace('/', os.sep) 1378 1379 self.backend.set_cover(book_id, path, data) 1380 for cc in self.cover_caches: 1381 cc.invalidate(book_id_data_map) 1382 return self._set_field('cover', { 1383 book_id:(0 if data is None else 1) for book_id, data in iteritems(book_id_data_map)}) 1384 1385 @write_api 1386 def add_cover_cache(self, cover_cache): 1387 if not callable(cover_cache.invalidate): 1388 raise ValueError('Cover caches must have an invalidate method') 1389 self.cover_caches.add(cover_cache) 1390 1391 @write_api 1392 def remove_cover_cache(self, cover_cache): 1393 self.cover_caches.discard(cover_cache) 1394 1395 @write_api 1396 def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False, 1397 set_title=True, set_authors=True, allow_case_change=False): 1398 ''' 1399 Set metadata for the book `id` from the `Metadata` object `mi` 1400 1401 Setting force_changes=True will force set_metadata to update fields even 1402 if mi contains empty values. In this case, 'None' is distinguished from 1403 'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is. 1404 The tags, identifiers, and cover attributes are special cases. Tags and 1405 identifiers cannot be set to None so they will always be replaced if 1406 force_changes is true. You must ensure that mi contains the values you 1407 want the book to have. Covers are always changed if a new cover is 1408 provided, but are never deleted. Also note that force_changes has no 1409 effect on setting title or authors. 1410 ''' 1411 dirtied = set() 1412 1413 try: 1414 # Handle code passing in an OPF object instead of a Metadata object 1415 mi = mi.to_book_metadata() 1416 except (AttributeError, TypeError): 1417 pass 1418 1419 def set_field(name, val): 1420 dirtied.update(self._set_field(name, {book_id:val}, do_path_update=False, allow_case_change=allow_case_change)) 1421 1422 path_changed = False 1423 if set_title and mi.title: 1424 path_changed = True 1425 set_field('title', mi.title) 1426 authors_changed = False 1427 if set_authors: 1428 path_changed = True 1429 if not mi.authors: 1430 mi.authors = [_('Unknown')] 1431 authors = [] 1432 for a in mi.authors: 1433 authors += string_to_authors(a) 1434 set_field('authors', authors) 1435 authors_changed = True 1436 1437 if path_changed: 1438 self._update_path({book_id}) 1439 1440 def protected_set_field(name, val): 1441 try: 1442 set_field(name, val) 1443 except: 1444 if ignore_errors: 1445 traceback.print_exc() 1446 else: 1447 raise 1448 1449 # force_changes has no effect on cover manipulation 1450 try: 1451 cdata = mi.cover_data[1] 1452 if cdata is None and isinstance(mi.cover, string_or_bytes) and mi.cover and os.access(mi.cover, os.R_OK): 1453 with lopen(mi.cover, 'rb') as f: 1454 cdata = f.read() or None 1455 if cdata is not None: 1456 self._set_cover({book_id: cdata}) 1457 except: 1458 if ignore_errors: 1459 traceback.print_exc() 1460 else: 1461 raise 1462 1463 try: 1464 with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode 1465 for field in ('rating', 'series_index', 'timestamp'): 1466 val = getattr(mi, field) 1467 if val is not None: 1468 protected_set_field(field, val) 1469 1470 val = mi.get('author_sort', None) 1471 if authors_changed and (not val or mi.is_null('author_sort')): 1472 val = self._author_sort_from_authors(mi.authors) 1473 if authors_changed or (force_changes and val is not None) or not mi.is_null('author_sort'): 1474 protected_set_field('author_sort', val) 1475 1476 for field in ('publisher', 'series', 'tags', 'comments', 1477 'languages', 'pubdate'): 1478 val = mi.get(field, None) 1479 if (force_changes and val is not None) or not mi.is_null(field): 1480 protected_set_field(field, val) 1481 1482 val = mi.get('title_sort', None) 1483 if (force_changes and val is not None) or not mi.is_null('title_sort'): 1484 protected_set_field('sort', val) 1485 1486 # identifiers will always be replaced if force_changes is True 1487 mi_idents = mi.get_identifiers() 1488 if force_changes: 1489 protected_set_field('identifiers', mi_idents) 1490 elif mi_idents: 1491 identifiers = self._field_for('identifiers', book_id, default_value={}) 1492 for key, val in iteritems(mi_idents): 1493 if val and val.strip(): # Don't delete an existing identifier 1494 identifiers[icu_lower(key)] = val 1495 protected_set_field('identifiers', identifiers) 1496 1497 user_mi = mi.get_all_user_metadata(make_copy=False) 1498 fm = self.field_metadata 1499 for key in user_mi: 1500 if (key in fm and user_mi[key]['datatype'] == fm[key]['datatype'] and ( 1501 user_mi[key]['datatype'] != 'text' or ( 1502 user_mi[key]['is_multiple'] == fm[key]['is_multiple']))): 1503 val = mi.get(key, None) 1504 if force_changes or val is not None: 1505 protected_set_field(key, val) 1506 idx = key + '_index' 1507 if idx in self.fields: 1508 extra = mi.get_extra(key) 1509 if extra is not None or force_changes: 1510 protected_set_field(idx, extra) 1511 except: 1512 # sqlite will rollback the entire transaction, thanks to the with 1513 # statement, so we have to re-read everything form the db to ensure 1514 # the db and Cache are in sync 1515 self._reload_from_db() 1516 raise 1517 return dirtied 1518 1519 def _do_add_format(self, book_id, fmt, stream, name=None, mtime=None): 1520 path = self._field_for('path', book_id) 1521 if path is None: 1522 # Theoretically, this should never happen, but apparently it 1523 # does: https://www.mobileread.com/forums/showthread.php?t=233353 1524 self._update_path({book_id}, mark_as_dirtied=False) 1525 path = self._field_for('path', book_id) 1526 1527 path = path.replace('/', os.sep) 1528 title = self._field_for('title', book_id, default_value=_('Unknown')) 1529 try: 1530 author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] 1531 except IndexError: 1532 author = _('Unknown') 1533 1534 size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path, name, mtime=mtime) 1535 return size, fname 1536 1537 @api 1538 def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): 1539 ''' 1540 Add a format to the specified book. Return True if the format was added successfully. 1541 1542 :param replace: If True replace existing format, otherwise if the format already exists, return False. 1543 :param run_hooks: If True, file type plugins are run on the format before and after being added. 1544 :param dbapi: Internal use only. 1545 ''' 1546 needs_close = False 1547 if run_hooks: 1548 # Run import plugins, the write lock is not held to cater for 1549 # broken plugins that might spin the event loop by popping up a 1550 # message in the GUI during the processing. 1551 npath = run_import_plugins(stream_or_path, fmt) 1552 fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() 1553 stream_or_path = lopen(npath, 'rb') 1554 needs_close = True 1555 fmt = check_ebook_format(stream_or_path, fmt) 1556 1557 with self.write_lock: 1558 if not self._has_id(book_id): 1559 raise NoSuchBook(book_id) 1560 fmt = (fmt or '').upper() 1561 self.format_metadata_cache[book_id].pop(fmt, None) 1562 try: 1563 name = self.fields['formats'].format_fname(book_id, fmt) 1564 except Exception: 1565 name = None 1566 1567 if name and not replace: 1568 return False 1569 1570 if hasattr(stream_or_path, 'read'): 1571 stream = stream_or_path 1572 else: 1573 stream = lopen(stream_or_path, 'rb') 1574 needs_close = True 1575 try: 1576 stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') 1577 size, fname = self._do_add_format(book_id, fmt, stream, name) 1578 finally: 1579 if needs_close: 1580 stream.close() 1581 del stream 1582 1583 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) 1584 self.fields['size'].table.update_sizes({book_id: max_size}) 1585 self._update_last_modified((book_id,)) 1586 self.event_dispatcher(EventType.format_added, book_id, fmt) 1587 1588 if run_hooks: 1589 # Run post import plugins, the write lock is released so the plugin 1590 # can call api without a locking violation. 1591 run_plugins_on_postimport(dbapi or self, book_id, fmt) 1592 stream_or_path.close() 1593 1594 return True 1595 1596 @write_api 1597 def remove_formats(self, formats_map, db_only=False): 1598 ''' 1599 Remove the specified formats from the specified books. 1600 1601 :param formats_map: A mapping of book_id to a list of formats to be removed from the book. 1602 :param db_only: If True, only remove the record for the format from the db, do not delete the actual format file from the filesystem. 1603 ''' 1604 table = self.fields['formats'].table 1605 formats_map = {book_id:frozenset((f or '').upper() for f in fmts) for book_id, fmts in iteritems(formats_map)} 1606 1607 for book_id, fmts in iteritems(formats_map): 1608 for fmt in fmts: 1609 self.format_metadata_cache[book_id].pop(fmt, None) 1610 1611 if not db_only: 1612 removes = defaultdict(set) 1613 for book_id, fmts in iteritems(formats_map): 1614 try: 1615 path = self._field_for('path', book_id).replace('/', os.sep) 1616 except: 1617 continue 1618 for fmt in fmts: 1619 try: 1620 name = self.fields['formats'].format_fname(book_id, fmt) 1621 except: 1622 continue 1623 if name and path: 1624 removes[book_id].add((fmt, name, path)) 1625 if removes: 1626 self.backend.remove_formats(removes) 1627 1628 size_map = table.remove_formats(formats_map, self.backend) 1629 self.fields['size'].table.update_sizes(size_map) 1630 self._update_last_modified(tuple(formats_map)) 1631 self.event_dispatcher(EventType.formats_removed, formats_map) 1632 1633 @read_api 1634 def get_next_series_num_for(self, series, field='series', current_indices=False): 1635 ''' 1636 Return the next series index for the specified series, taking into account the various preferences that 1637 control next series number generation. 1638 1639 :param field: The series-like field (defaults to the builtin series column) 1640 :param current_indices: If True, returns a mapping of book_id to current series_index value instead. 1641 ''' 1642 books = () 1643 sf = self.fields[field] 1644 if series: 1645 q = icu_lower(series) 1646 for val, book_ids in sf.iter_searchable_values(self._get_proxy_metadata, frozenset(self._all_book_ids())): 1647 if q == icu_lower(val): 1648 books = book_ids 1649 break 1650 idf = sf.index_field 1651 index_map = {book_id:self._fast_field_for(idf, book_id, default_value=1.0) for book_id in books} 1652 if current_indices: 1653 return index_map 1654 series_indices = sorted(itervalues(index_map)) 1655 return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) 1656 1657 @read_api 1658 def author_sort_from_authors(self, authors, key_func=icu_lower): 1659 '''Given a list of authors, return the author_sort string for the authors, 1660 preferring the author sort associated with the author over the computed 1661 string. ''' 1662 table = self.fields['authors'].table 1663 result = [] 1664 rmap = {key_func(v):k for k, v in iteritems(table.id_map)} 1665 for aut in authors: 1666 aid = rmap.get(key_func(aut), None) 1667 result.append(author_to_author_sort(aut) if aid is None else table.asort_map[aid]) 1668 return ' & '.join(_f for _f in result if _f) 1669 1670 @read_api 1671 def data_for_has_book(self): 1672 ''' Return data suitable for use in :meth:`has_book`. This can be used for an 1673 implementation of :meth:`has_book` in a worker process without access to the 1674 db. ''' 1675 try: 1676 return {icu_lower(title) for title in itervalues(self.fields['title'].table.book_col_map)} 1677 except TypeError: 1678 # Some non-unicode titles in the db 1679 return {icu_lower(as_unicode(title)) for title in itervalues(self.fields['title'].table.book_col_map)} 1680 1681 @read_api 1682 def has_book(self, mi): 1683 ''' Return True iff the database contains an entry with the same title 1684 as the passed in Metadata object. The comparison is case-insensitive. 1685 See also :meth:`data_for_has_book`. ''' 1686 title = mi.title 1687 if title: 1688 if isbytestring(title): 1689 title = title.decode(preferred_encoding, 'replace') 1690 q = icu_lower(title).strip() 1691 for title in itervalues(self.fields['title'].table.book_col_map): 1692 if q == icu_lower(title): 1693 return True 1694 return False 1695 1696 @read_api 1697 def has_id(self, book_id): 1698 ' Return True iff the specified book_id exists in the db ''' 1699 return book_id in self.fields['title'].table.book_col_map 1700 1701 @write_api 1702 def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False): 1703 if mi.tags: 1704 mi.tags = list(mi.tags) 1705 if apply_import_tags: 1706 _add_newbook_tag(mi) 1707 _add_default_custom_column_values(mi, self.field_metadata) 1708 if not add_duplicates and self._has_book(mi): 1709 return 1710 series_index = (self._get_next_series_num_for(mi.series) if mi.series_index is None else mi.series_index) 1711 try: 1712 series_index = float(series_index) 1713 except Exception: 1714 try: 1715 series_index = float(self._get_next_series_num_for(mi.series)) 1716 except Exception: 1717 series_index = 1.0 1718 if not mi.authors: 1719 mi.authors = (_('Unknown'),) 1720 aus = mi.author_sort if not mi.is_null('author_sort') else self._author_sort_from_authors(mi.authors) 1721 mi.title = mi.title or _('Unknown') 1722 if isbytestring(aus): 1723 aus = aus.decode(preferred_encoding, 'replace') 1724 if isbytestring(mi.title): 1725 mi.title = mi.title.decode(preferred_encoding, 'replace') 1726 if force_id is None: 1727 self.backend.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)', 1728 (mi.title, series_index, aus)) 1729 else: 1730 self.backend.execute('INSERT INTO books(id, title, series_index, author_sort) VALUES (?, ?, ?, ?)', 1731 (force_id, mi.title, series_index, aus)) 1732 book_id = self.backend.last_insert_rowid() 1733 self.event_dispatcher(EventType.book_created, book_id) 1734 1735 mi.timestamp = utcnow() if mi.timestamp is None else mi.timestamp 1736 mi.pubdate = UNDEFINED_DATE if mi.pubdate is None else mi.pubdate 1737 if cover is not None: 1738 mi.cover, mi.cover_data = None, (None, cover) 1739 self._set_metadata(book_id, mi, ignore_errors=True) 1740 if preserve_uuid and mi.uuid: 1741 self._set_field('uuid', {book_id:mi.uuid}) 1742 # Update the caches for fields from the books table 1743 self.fields['size'].table.book_col_map[book_id] = 0 1744 row = next(self.backend.execute('SELECT sort, series_index, author_sort, uuid, has_cover FROM books WHERE id=?', (book_id,))) 1745 for field, val in zip(('sort', 'series_index', 'author_sort', 'uuid', 'cover'), row): 1746 if field == 'cover': 1747 val = bool(val) 1748 elif field == 'uuid': 1749 self.fields[field].table.uuid_to_id_map[val] = book_id 1750 self.fields[field].table.book_col_map[book_id] = val 1751 1752 return book_id 1753 1754 @api 1755 def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None): 1756 ''' 1757 Add the specified books to the library. Books should be an iterable of 1758 2-tuples, each 2-tuple of the form :code:`(mi, format_map)` where mi is a 1759 Metadata object and format_map is a dictionary of the form :code:`{fmt: path_or_stream}`, 1760 for example: :code:`{'EPUB': '/path/to/file.epub'}`. 1761 1762 Returns a pair of lists: :code:`ids, duplicates`. ``ids`` contains the book ids for all newly created books in the 1763 database. ``duplicates`` contains the :code:`(mi, format_map)` for all books that already exist in the database 1764 as per the simple duplicate detection heuristic used by :meth:`has_book`. 1765 ''' 1766 duplicates, ids = [], [] 1767 fmt_map = {} 1768 for mi, format_map in books: 1769 book_id = self.create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) 1770 if book_id is None: 1771 duplicates.append((mi, format_map)) 1772 else: 1773 ids.append(book_id) 1774 for fmt, stream_or_path in iteritems(format_map): 1775 if self.add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks): 1776 fmt_map[fmt.lower()] = getattr(stream_or_path, 'name', stream_or_path) or '<stream>' 1777 run_plugins_on_postadd(dbapi or self, book_id, fmt_map) 1778 return ids, duplicates 1779 1780 @write_api 1781 def remove_books(self, book_ids, permanent=False): 1782 ''' Remove the books specified by the book_ids from the database and delete 1783 their format files. If ``permanent`` is False, then the format files 1784 are placed in the recycle bin. ''' 1785 path_map = {} 1786 for book_id in book_ids: 1787 try: 1788 path = self._field_for('path', book_id).replace('/', os.sep) 1789 except: 1790 path = None 1791 path_map[book_id] = path 1792 if iswindows: 1793 paths = (x.replace(os.sep, '/') for x in itervalues(path_map) if x) 1794 self.backend.windows_check_if_files_in_use(paths) 1795 1796 self.backend.remove_books(path_map, permanent=permanent) 1797 for field in itervalues(self.fields): 1798 try: 1799 table = field.table 1800 except AttributeError: 1801 continue # Some fields like ondevice do not have tables 1802 else: 1803 table.remove_books(book_ids, self.backend) 1804 self._search_api.discard_books(book_ids) 1805 self._clear_caches(book_ids=book_ids, template_cache=False, search_cache=False) 1806 for cc in self.cover_caches: 1807 cc.invalidate(book_ids) 1808 self.event_dispatcher(EventType.books_removed, book_ids) 1809 1810 @read_api 1811 def author_sort_strings_for_books(self, book_ids): 1812 val_map = {} 1813 for book_id in book_ids: 1814 authors = self._field_ids_for('authors', book_id) 1815 adata = self._author_data(authors) 1816 val_map[book_id] = tuple(adata[aid]['sort'] for aid in authors) 1817 return val_map 1818 1819 @write_api 1820 def rename_items(self, field, item_id_to_new_name_map, change_index=True, restrict_to_book_ids=None): 1821 ''' 1822 Rename items from a many-one or many-many field such as tags or series. 1823 1824 :param change_index: When renaming in a series-like field also change the series_index values. 1825 :param restrict_to_book_ids: An optional set of book ids for which the rename is to be performed, defaults to all books. 1826 ''' 1827 1828 f = self.fields[field] 1829 affected_books = set() 1830 try: 1831 sv = f.metadata['is_multiple']['ui_to_list'] 1832 except (TypeError, KeyError, AttributeError): 1833 sv = None 1834 1835 if restrict_to_book_ids is not None: 1836 # We have a VL. Only change the item name for those books 1837 if not isinstance(restrict_to_book_ids, (Set, MutableSet)): 1838 restrict_to_book_ids = frozenset(restrict_to_book_ids) 1839 id_map = {} 1840 default_process_map = {} 1841 for old_id, new_name in iteritems(item_id_to_new_name_map): 1842 new_names = tuple(x.strip() for x in new_name.split(sv)) if sv else (new_name,) 1843 # Get a list of books in the VL with the item 1844 books_with_id = f.books_for(old_id) 1845 books_to_process = books_with_id & restrict_to_book_ids 1846 if len(books_with_id) == len(books_to_process): 1847 # All the books with the ID are in the VL, so we can use 1848 # the normal processing 1849 default_process_map[old_id] = new_name 1850 elif books_to_process: 1851 affected_books.update(books_to_process) 1852 newvals = {} 1853 for book_id in books_to_process: 1854 # Get the current values, remove the one being renamed, then add 1855 # the new value(s) back. 1856 vals = self._field_for(field, book_id) 1857 # Check for is_multiple 1858 if isinstance(vals, tuple): 1859 # We must preserve order. 1860 vals = list(vals) 1861 # Don't need to worry about case here because we 1862 # are fetching its one-true spelling. But lets be 1863 # careful anyway 1864 try: 1865 dex = vals.index(self._get_item_name(field, old_id)) 1866 # This can put the name back with a different case 1867 vals[dex] = new_names[0] 1868 # now add any other items if they aren't already there 1869 if len(new_names) > 1: 1870 set_vals = {icu_lower(x) for x in vals} 1871 for v in new_names[1:]: 1872 lv = icu_lower(v) 1873 if lv not in set_vals: 1874 vals.append(v) 1875 set_vals.add(lv) 1876 newvals[book_id] = vals 1877 except Exception: 1878 traceback.print_exc() 1879 else: 1880 newvals[book_id] = new_names[0] 1881 # Allow case changes 1882 self._set_field(field, newvals) 1883 id_map[old_id] = self._get_item_id(field, new_names[0]) 1884 if default_process_map: 1885 ab, idm = self._rename_items(field, default_process_map, change_index=change_index) 1886 affected_books.update(ab) 1887 id_map.update(idm) 1888 self.event_dispatcher(EventType.items_renamed, field, affected_books, id_map) 1889 return affected_books, id_map 1890 1891 try: 1892 func = f.table.rename_item 1893 except AttributeError: 1894 raise ValueError('Cannot rename items for one-one fields: %s' % field) 1895 moved_books = set() 1896 id_map = {} 1897 for item_id, new_name in iteritems(item_id_to_new_name_map): 1898 new_names = tuple(x.strip() for x in new_name.split(sv)) if sv else (new_name,) 1899 books, new_id = func(item_id, new_names[0], self.backend) 1900 affected_books.update(books) 1901 id_map[item_id] = new_id 1902 if new_id != item_id: 1903 moved_books.update(books) 1904 if len(new_names) > 1: 1905 # Add the extra items to the books 1906 extra = new_names[1:] 1907 self._set_field(field, {book_id:self._fast_field_for(f, book_id) + extra for book_id in books}) 1908 1909 if affected_books: 1910 if field == 'authors': 1911 self._set_field('author_sort', 1912 {k:' & '.join(v) for k, v in iteritems(self._author_sort_strings_for_books(affected_books))}) 1913 self._update_path(affected_books, mark_as_dirtied=False) 1914 elif change_index and hasattr(f, 'index_field') and tweaks['series_index_auto_increment'] != 'no_change': 1915 for book_id in moved_books: 1916 self._set_field(f.index_field.name, {book_id:self._get_next_series_num_for(self._fast_field_for(f, book_id), field=field)}) 1917 self._mark_as_dirty(affected_books) 1918 self.event_dispatcher(EventType.items_renamed, field, affected_books, id_map) 1919 return affected_books, id_map 1920 1921 @write_api 1922 def remove_items(self, field, item_ids, restrict_to_book_ids=None): 1923 ''' Delete all items in the specified field with the specified ids. 1924 Returns the set of affected book ids. ``restrict_to_book_ids`` is an 1925 optional set of books ids. If specified the items will only be removed 1926 from those books. ''' 1927 field = self.fields[field] 1928 if restrict_to_book_ids is not None and not isinstance(restrict_to_book_ids, (MutableSet, Set)): 1929 restrict_to_book_ids = frozenset(restrict_to_book_ids) 1930 affected_books = field.table.remove_items(item_ids, self.backend, 1931 restrict_to_book_ids=restrict_to_book_ids) 1932 if affected_books: 1933 if hasattr(field, 'index_field'): 1934 self._set_field(field.index_field.name, {bid:1.0 for bid in affected_books}) 1935 else: 1936 self._mark_as_dirty(affected_books) 1937 self.event_dispatcher(EventType.items_removed, field, affected_books, item_ids) 1938 return affected_books 1939 1940 @write_api 1941 def add_custom_book_data(self, name, val_map, delete_first=False): 1942 ''' Add data for name where val_map is a map of book_ids to values. If 1943 delete_first is True, all previously stored data for name will be 1944 removed. ''' 1945 missing = frozenset(val_map) - self._all_book_ids() 1946 if missing: 1947 raise ValueError('add_custom_book_data: no such book_ids: %d'%missing) 1948 self.backend.add_custom_data(name, val_map, delete_first) 1949 1950 @read_api 1951 def get_custom_book_data(self, name, book_ids=(), default=None): 1952 ''' Get data for name. By default returns data for all book_ids, pass 1953 in a list of book ids if you only want some data. Returns a map of 1954 book_id to values. If a particular value could not be decoded, uses 1955 default for it. ''' 1956 return self.backend.get_custom_book_data(name, book_ids, default) 1957 1958 @write_api 1959 def delete_custom_book_data(self, name, book_ids=()): 1960 ''' Delete data for name. By default deletes all data, if you only want 1961 to delete data for some book ids, pass in a list of book ids. ''' 1962 self.backend.delete_custom_book_data(name, book_ids) 1963 1964 @read_api 1965 def get_ids_for_custom_book_data(self, name): 1966 ''' Return the set of book ids for which name has data. ''' 1967 return self.backend.get_ids_for_custom_book_data(name) 1968 1969 @read_api 1970 def conversion_options(self, book_id, fmt='PIPE'): 1971 return self.backend.conversion_options(book_id, fmt) 1972 1973 @read_api 1974 def has_conversion_options(self, ids, fmt='PIPE'): 1975 return self.backend.has_conversion_options(ids, fmt) 1976 1977 @write_api 1978 def delete_conversion_options(self, book_ids, fmt='PIPE'): 1979 return self.backend.delete_conversion_options(book_ids, fmt) 1980 1981 @write_api 1982 def set_conversion_options(self, options, fmt='PIPE'): 1983 ''' options must be a map of the form {book_id:conversion_options} ''' 1984 return self.backend.set_conversion_options(options, fmt) 1985 1986 @write_api 1987 def refresh_format_cache(self): 1988 self.fields['formats'].table.read(self.backend) 1989 self.format_metadata_cache.clear() 1990 1991 @write_api 1992 def refresh_ondevice(self): 1993 self.fields['ondevice'].clear_caches() 1994 self.clear_search_caches() 1995 self.clear_composite_caches() 1996 1997 @read_api 1998 def books_matching_device_book(self, lpath): 1999 ans = set() 2000 for book_id, (_, _, _, _, lpaths) in self.fields['ondevice'].cache.items(): 2001 if lpath in lpaths: 2002 ans.add(book_id) 2003 return ans 2004 2005 @read_api 2006 def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None): 2007 ''' 2008 Return the ids of all books having the tag ``tag`` that are older than 2009 the specified time. tag comparison is case insensitive. 2010 2011 :param delta: A timedelta object or None. If None, then all ids with 2012 the tag are returned. 2013 2014 :param must_have_tag: If not None the list of matches will be 2015 restricted to books that have this tag 2016 2017 :param must_have_authors: A list of authors. If not None the list of 2018 matches will be restricted to books that have these authors (case 2019 insensitive). 2020 2021 ''' 2022 tag_map = {icu_lower(v):k for k, v in iteritems(self._get_id_map('tags'))} 2023 tag = icu_lower(tag.strip()) 2024 mht = icu_lower(must_have_tag.strip()) if must_have_tag else None 2025 tag_id, mht_id = tag_map.get(tag, None), tag_map.get(mht, None) 2026 ans = set() 2027 if mht_id is None and mht: 2028 return ans 2029 if tag_id is not None: 2030 tagged_books = self._books_for_field('tags', tag_id) 2031 if mht_id is not None and tagged_books: 2032 tagged_books = tagged_books.intersection(self._books_for_field('tags', mht_id)) 2033 if tagged_books: 2034 if must_have_authors is not None: 2035 amap = {icu_lower(v):k for k, v in iteritems(self._get_id_map('authors'))} 2036 books = None 2037 for author in must_have_authors: 2038 abooks = self._books_for_field('authors', amap.get(icu_lower(author), None)) 2039 books = abooks if books is None else books.intersection(abooks) 2040 if not books: 2041 break 2042 tagged_books = tagged_books.intersection(books or set()) 2043 if delta is None: 2044 ans = tagged_books 2045 else: 2046 now = nowf() 2047 for book_id in tagged_books: 2048 ts = self._field_for('timestamp', book_id) 2049 if (now - ts) > delta: 2050 ans.add(book_id) 2051 return ans 2052 2053 @write_api 2054 def set_sort_for_authors(self, author_id_to_sort_map, update_books=True): 2055 sort_map = self.fields['authors'].table.set_sort_names(author_id_to_sort_map, self.backend) 2056 changed_books = set() 2057 if update_books: 2058 val_map = {} 2059 for author_id in sort_map: 2060 books = self._books_for_field('authors', author_id) 2061 changed_books |= books 2062 for book_id in books: 2063 authors = self._field_ids_for('authors', book_id) 2064 adata = self._author_data(authors) 2065 sorts = [adata[x]['sort'] for x in authors] 2066 val_map[book_id] = ' & '.join(sorts) 2067 if val_map: 2068 self._set_field('author_sort', val_map) 2069 if changed_books: 2070 self._mark_as_dirty(changed_books) 2071 return changed_books 2072 2073 @write_api 2074 def set_link_for_authors(self, author_id_to_link_map): 2075 link_map = self.fields['authors'].table.set_links(author_id_to_link_map, self.backend) 2076 changed_books = set() 2077 for author_id in link_map: 2078 changed_books |= self._books_for_field('authors', author_id) 2079 if changed_books: 2080 self._mark_as_dirty(changed_books) 2081 return changed_books 2082 2083 @read_api 2084 def lookup_by_uuid(self, uuid): 2085 return self.fields['uuid'].table.lookup_by_uuid(uuid) 2086 2087 @write_api 2088 def delete_custom_column(self, label=None, num=None): 2089 self.backend.delete_custom_column(label, num) 2090 2091 @write_api 2092 def create_custom_column(self, label, name, datatype, is_multiple, editable=True, display={}): 2093 return self.backend.create_custom_column(label, name, datatype, is_multiple, editable=editable, display=display) 2094 2095 @write_api 2096 def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, 2097 display=None, update_last_modified=False): 2098 changed = self.backend.set_custom_column_metadata(num, name=name, label=label, is_editable=is_editable, display=display) 2099 if changed: 2100 if update_last_modified: 2101 self._update_last_modified(self._all_book_ids()) 2102 else: 2103 self.backend.prefs.set('update_all_last_mod_dates_on_start', True) 2104 return changed 2105 2106 @read_api 2107 def get_books_for_category(self, category, item_id_or_composite_value): 2108 f = self.fields[category] 2109 if hasattr(f, 'get_books_for_val'): 2110 # Composite field 2111 return f.get_books_for_val(item_id_or_composite_value, self._get_proxy_metadata, self._all_book_ids()) 2112 return self._books_for_field(f.name, int(item_id_or_composite_value)) 2113 2114 @read_api 2115 def data_for_find_identical_books(self): 2116 ''' Return data that can be used to implement 2117 :meth:`find_identical_books` in a worker process without access to the 2118 db. See db.utils for an implementation. ''' 2119 at = self.fields['authors'].table 2120 author_map = defaultdict(set) 2121 for aid, author in iteritems(at.id_map): 2122 author_map[icu_lower(author)].add(aid) 2123 return (author_map, at.col_book_map.copy(), self.fields['title'].table.book_col_map.copy(), self.fields['languages'].book_value_map.copy()) 2124 2125 @read_api 2126 def update_data_for_find_identical_books(self, book_id, data): 2127 author_map, author_book_map, title_map, lang_map = data 2128 title_map[book_id] = self._field_for('title', book_id) 2129 lang_map[book_id] = self._field_for('languages', book_id) 2130 at = self.fields['authors'].table 2131 for aid in at.book_col_map.get(book_id, ()): 2132 author_map[icu_lower(at.id_map[aid])].add(aid) 2133 try: 2134 author_book_map[aid].add(book_id) 2135 except KeyError: 2136 author_book_map[aid] = {book_id} 2137 2138 @read_api 2139 def find_identical_books(self, mi, search_restriction='', book_ids=None): 2140 ''' Finds books that have a superset of the authors in mi and the same 2141 title (title is fuzzy matched). See also :meth:`data_for_find_identical_books`. ''' 2142 from calibre.db.utils import fuzzy_title 2143 identical_book_ids = set() 2144 langq = tuple(x for x in map(canonicalize_lang, mi.languages or ()) if x and x != 'und') 2145 if mi.authors: 2146 try: 2147 quathors = mi.authors[:20] # Too many authors causes parsing of the search expression to fail 2148 query = ' and '.join('authors:"=%s"'%(a.replace('"', '')) for a in quathors) 2149 qauthors = mi.authors[20:] 2150 except ValueError: 2151 return identical_book_ids 2152 try: 2153 book_ids = self._search(query, restriction=search_restriction, book_ids=book_ids) 2154 except: 2155 traceback.print_exc() 2156 return identical_book_ids 2157 if qauthors and book_ids: 2158 matches = set() 2159 qauthors = {icu_lower(x) for x in qauthors} 2160 for book_id in book_ids: 2161 aut = self._field_for('authors', book_id) 2162 if aut: 2163 aut = {icu_lower(x) for x in aut} 2164 if aut.issuperset(qauthors): 2165 matches.add(book_id) 2166 book_ids = matches 2167 2168 for book_id in book_ids: 2169 fbook_title = self._field_for('title', book_id) 2170 fbook_title = fuzzy_title(fbook_title) 2171 mbook_title = fuzzy_title(mi.title) 2172 if fbook_title == mbook_title: 2173 bl = self._field_for('languages', book_id) 2174 if not langq or not bl or bl == langq: 2175 identical_book_ids.add(book_id) 2176 return identical_book_ids 2177 2178 @read_api 2179 def get_top_level_move_items(self): 2180 all_paths = {self._field_for('path', book_id).partition('/')[0] for book_id in self._all_book_ids()} 2181 return self.backend.get_top_level_move_items(all_paths) 2182 2183 @write_api 2184 def move_library_to(self, newloc, progress=None, abort=None): 2185 def progress_callback(item_name, item_count, total): 2186 try: 2187 if progress is not None: 2188 progress(item_name, item_count, total) 2189 except Exception: 2190 import traceback 2191 traceback.print_exc() 2192 2193 all_paths = {self._field_for('path', book_id).partition('/')[0] for book_id in self._all_book_ids()} 2194 self.backend.move_library_to(all_paths, newloc, progress=progress_callback, abort=abort) 2195 2196 @read_api 2197 def saved_search_names(self): 2198 return self._search_api.saved_searches.names() 2199 2200 @read_api 2201 def saved_search_lookup(self, name): 2202 return self._search_api.saved_searches.lookup(name) 2203 2204 @write_api 2205 def saved_search_set_all(self, smap): 2206 self._search_api.saved_searches.set_all(smap) 2207 self._clear_search_caches() 2208 2209 @write_api 2210 def saved_search_delete(self, name): 2211 self._search_api.saved_searches.delete(name) 2212 self._clear_search_caches() 2213 2214 @write_api 2215 def saved_search_add(self, name, val): 2216 self._search_api.saved_searches.add(name, val) 2217 2218 @write_api 2219 def saved_search_rename(self, old_name, new_name): 2220 self._search_api.saved_searches.rename(old_name, new_name) 2221 self._clear_search_caches() 2222 2223 @write_api 2224 def change_search_locations(self, newlocs): 2225 self._search_api.change_locations(newlocs) 2226 2227 @write_api 2228 def refresh_search_locations(self): 2229 self._search_api.change_locations(self.field_metadata.get_search_terms()) 2230 2231 @write_api 2232 def dump_and_restore(self, callback=None, sql=None): 2233 return self.backend.dump_and_restore(callback=callback, sql=sql) 2234 2235 @write_api 2236 def vacuum(self): 2237 self.backend.vacuum() 2238 2239 @write_api 2240 def close(self): 2241 self.event_dispatcher.close() 2242 from calibre.customize.ui import available_library_closed_plugins 2243 for plugin in available_library_closed_plugins(): 2244 try: 2245 plugin.run(self) 2246 except Exception: 2247 import traceback 2248 traceback.print_exc() 2249 self.backend.close() 2250 2251 @property 2252 def is_closed(self): 2253 return self.backend.is_closed 2254 2255 @write_api 2256 def restore_book(self, book_id, mi, last_modified, path, formats, annotations=()): 2257 ''' Restore the book entry in the database for a book that already exists on the filesystem ''' 2258 cover = mi.cover 2259 mi.cover = None 2260 self._create_book_entry(mi, add_duplicates=True, 2261 force_id=book_id, apply_import_tags=False, preserve_uuid=True) 2262 self._update_last_modified((book_id,), last_modified) 2263 if cover and os.path.exists(cover): 2264 self._set_field('cover', {book_id:1}) 2265 self.backend.restore_book(book_id, path, formats) 2266 if annotations: 2267 self._restore_annotations(book_id, annotations) 2268 2269 @read_api 2270 def virtual_libraries_for_books(self, book_ids, virtual_fields=None): 2271 # use a primitive lock to ensure that only one thread is updating 2272 # the cache and that recursive calls don't do the update. This 2273 # method can recurse via self._search() 2274 with try_lock(self.vls_cache_lock) as got_lock: 2275 # Using a list is slightly faster than a set. 2276 c = defaultdict(list) 2277 if not got_lock: 2278 # We get here if resolving the books in a VL triggers another VL 2279 # calculation. This can be 'real' recursion, in which case the 2280 # eventual answer will be wrong. It can also be a search using 2281 # a location of 'all' that causes evaluation of a composite that 2282 # references virtual_libraries(). If the composite isn't used in a 2283 # VL then the eventual answer will be correct because get_metadata 2284 # will clear the caches. 2285 return c 2286 if self.vls_for_books_cache is None: 2287 self.vls_for_books_cache_is_loading = True 2288 libraries = self._pref('virtual_libraries', {}) 2289 for lib, expr in libraries.items(): 2290 book = None 2291 try: 2292 for book in self._search(expr, virtual_fields=virtual_fields): 2293 c[book].append(lib) 2294 except Exception as e: 2295 if book: 2296 c[book].append(_('[Error in Virtual library {0}: {1}]').format(lib, str(e))) 2297 self.vls_for_books_cache = {b:tuple(sorted(libs, key=sort_key)) for b, libs in c.items()} 2298 if not book_ids: 2299 book_ids = self._all_book_ids() 2300 # book_ids is usually 1 long. The loop will be faster than a comprehension 2301 r = {} 2302 default = () 2303 for b in book_ids: 2304 r[b] = self.vls_for_books_cache.get(b, default) 2305 return r 2306 2307 @read_api 2308 def user_categories_for_books(self, book_ids, proxy_metadata_map=None): 2309 ''' Return the user categories for the specified books. 2310 proxy_metadata_map is optional and is useful for a performance boost, 2311 in contexts where a ProxyMetadata object for the books already exists. 2312 It should be a mapping of book_ids to their corresponding ProxyMetadata 2313 objects. 2314 ''' 2315 user_cats = self._pref('user_categories', {}) 2316 pmm = proxy_metadata_map or {} 2317 ans = {} 2318 2319 for book_id in book_ids: 2320 proxy_metadata = pmm.get(book_id) or self._get_proxy_metadata(book_id) 2321 user_cat_vals = ans[book_id] = {} 2322 for ucat, categories in iteritems(user_cats): 2323 user_cat_vals[ucat] = res = [] 2324 for name, cat, ign in categories: 2325 try: 2326 field_obj = self.fields[cat] 2327 except KeyError: 2328 continue 2329 2330 if field_obj.is_composite: 2331 v = field_obj.get_value_with_cache(book_id, lambda x:proxy_metadata) 2332 else: 2333 v = self._fast_field_for(field_obj, book_id) 2334 2335 if isinstance(v, (list, tuple)): 2336 if name in v: 2337 res.append([name, cat]) 2338 elif name == v: 2339 res.append([name, cat]) 2340 return ans 2341 2342 @write_api 2343 def embed_metadata(self, book_ids, only_fmts=None, report_error=None, report_progress=None): 2344 ''' Update metadata in all formats of the specified book_ids to current metadata in the database. ''' 2345 field = self.fields['formats'] 2346 from calibre.customize.ui import apply_null_metadata 2347 from calibre.ebooks.metadata.meta import set_metadata 2348 from calibre.ebooks.metadata.opf2 import pretty_print 2349 if only_fmts: 2350 only_fmts = {f.lower() for f in only_fmts} 2351 2352 def doit(fmt, mi, stream): 2353 with apply_null_metadata, pretty_print: 2354 set_metadata(stream, mi, stream_type=fmt, report_error=report_error) 2355 stream.seek(0, os.SEEK_END) 2356 return stream.tell() 2357 2358 for i, book_id in enumerate(book_ids): 2359 fmts = field.table.book_col_map.get(book_id, ()) 2360 if not fmts: 2361 continue 2362 mi = self.get_metadata(book_id, get_cover=True, cover_as_data=True) 2363 try: 2364 path = self._field_for('path', book_id).replace('/', os.sep) 2365 except: 2366 continue 2367 for fmt in fmts: 2368 if only_fmts is not None and fmt.lower() not in only_fmts: 2369 continue 2370 try: 2371 name = self.fields['formats'].format_fname(book_id, fmt) 2372 except: 2373 continue 2374 if name and path: 2375 new_size = self.backend.apply_to_format(book_id, path, name, fmt, partial(doit, fmt, mi)) 2376 if new_size is not None: 2377 self.format_metadata_cache[book_id].get(fmt, {})['size'] = new_size 2378 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, name, new_size, self.backend) 2379 self.fields['size'].table.update_sizes({book_id: max_size}) 2380 if report_progress is not None: 2381 report_progress(i+1, len(book_ids), mi) 2382 2383 @read_api 2384 def get_last_read_positions(self, book_id, fmt, user): 2385 fmt = fmt.upper() 2386 ans = [] 2387 for device, cfi, epoch, pos_frac in self.backend.execute( 2388 'SELECT device,cfi,epoch,pos_frac FROM last_read_positions WHERE book=? AND format=? AND user=?', 2389 (book_id, fmt, user)): 2390 ans.append({'device':device, 'cfi': cfi, 'epoch':epoch, 'pos_frac':pos_frac}) 2391 return ans 2392 2393 @write_api 2394 def set_last_read_position(self, book_id, fmt, user='_', device='_', cfi=None, epoch=None, pos_frac=0): 2395 fmt = fmt.upper() 2396 device = device or '_' 2397 user = user or '_' 2398 if not cfi: 2399 self.backend.execute( 2400 'DELETE FROM last_read_positions WHERE book=? AND format=? AND user=? AND device=?', 2401 (book_id, fmt, user, device)) 2402 else: 2403 self.backend.execute( 2404 'INSERT OR REPLACE INTO last_read_positions(book,format,user,device,cfi,epoch,pos_frac) VALUES (?,?,?,?,?,?,?)', 2405 (book_id, fmt, user, device, cfi, epoch or time(), pos_frac)) 2406 2407 @read_api 2408 def export_library(self, library_key, exporter, progress=None, abort=None): 2409 from polyglot.binary import as_hex_unicode 2410 key_prefix = as_hex_unicode(library_key) 2411 book_ids = self._all_book_ids() 2412 total = len(book_ids) + 1 2413 format_metadata = {} 2414 if progress is not None: 2415 progress('metadata.db', 0, total) 2416 pt = PersistentTemporaryFile('-export.db') 2417 pt.close() 2418 self.backend.backup_database(pt.name) 2419 dbkey = key_prefix + ':::' + 'metadata.db' 2420 with lopen(pt.name, 'rb') as f: 2421 exporter.add_file(f, dbkey) 2422 os.remove(pt.name) 2423 metadata = {'format_data':format_metadata, 'metadata.db':dbkey, 'total':total} 2424 for i, book_id in enumerate(book_ids): 2425 if abort is not None and abort.is_set(): 2426 return 2427 if progress is not None: 2428 progress(self._field_for('title', book_id), i + 1, total) 2429 format_metadata[book_id] = {} 2430 for fmt in self._formats(book_id): 2431 mdata = self.format_metadata(book_id, fmt) 2432 key = '%s:%s:%s' % (key_prefix, book_id, fmt) 2433 format_metadata[book_id][fmt] = key 2434 with exporter.start_file(key, mtime=mdata.get('mtime')) as dest: 2435 self._copy_format_to(book_id, fmt, dest, report_file_size=dest.ensure_space) 2436 cover_key = '%s:%s:%s' % (key_prefix, book_id, '.cover') 2437 with exporter.start_file(cover_key) as dest: 2438 if not self.copy_cover_to(book_id, dest, report_file_size=dest.ensure_space): 2439 dest.discard() 2440 else: 2441 format_metadata[book_id]['.cover'] = cover_key 2442 exporter.set_metadata(library_key, metadata) 2443 if progress is not None: 2444 progress(_('Completed'), total, total) 2445 2446 @read_api 2447 def annotations_map_for_book(self, book_id, fmt, user_type='local', user='viewer'): 2448 ans = {} 2449 for annot in self.backend.annotations_for_book(book_id, fmt, user_type, user): 2450 ans.setdefault(annot['type'], []).append(annot) 2451 return ans 2452 2453 @read_api 2454 def all_annotations_for_book(self, book_id): 2455 return tuple(self.backend.all_annotations_for_book(book_id)) 2456 2457 @read_api 2458 def annotation_count_for_book(self, book_id): 2459 return self.backend.annotation_count_for_book(book_id) 2460 2461 @read_api 2462 def all_annotation_users(self): 2463 return tuple(self.backend.all_annotation_users()) 2464 2465 @read_api 2466 def all_annotation_types(self): 2467 return tuple(self.backend.all_annotation_types()) 2468 2469 @read_api 2470 def all_annotations(self, restrict_to_user=None, limit=None, annotation_type=None, ignore_removed=False, restrict_to_book_ids=None): 2471 return tuple(self.backend.all_annotations(restrict_to_user, limit, annotation_type, ignore_removed, restrict_to_book_ids)) 2472 2473 @read_api 2474 def search_annotations( 2475 self, 2476 fts_engine_query, 2477 use_stemming=True, 2478 highlight_start=None, 2479 highlight_end=None, 2480 snippet_size=None, 2481 annotation_type=None, 2482 restrict_to_book_ids=None, 2483 restrict_to_user=None, 2484 ignore_removed=False 2485 ): 2486 return tuple(self.backend.search_annotations( 2487 fts_engine_query, use_stemming, highlight_start, highlight_end, 2488 snippet_size, annotation_type, restrict_to_book_ids, restrict_to_user, 2489 ignore_removed 2490 )) 2491 2492 @write_api 2493 def delete_annotations(self, annot_ids): 2494 self.backend.delete_annotations(annot_ids) 2495 2496 @write_api 2497 def update_annotations(self, annot_id_map): 2498 self.backend.update_annotations(annot_id_map) 2499 2500 @write_api 2501 def restore_annotations(self, book_id, annotations): 2502 from calibre.utils.date import EPOCH 2503 from calibre.utils.iso8601 import parse_iso8601 2504 umap = defaultdict(list) 2505 for adata in annotations: 2506 key = adata['user_type'], adata['user'], adata['format'] 2507 a = adata['annotation'] 2508 ts = (parse_iso8601(a['timestamp']) - EPOCH).total_seconds() 2509 umap[key].append((a, ts)) 2510 for (user_type, user, fmt), annots_list in iteritems(umap): 2511 self._set_annotations_for_book(book_id, fmt, annots_list, user_type=user_type, user=user) 2512 2513 @write_api 2514 def set_annotations_for_book(self, book_id, fmt, annots_list, user_type='local', user='viewer'): 2515 self.backend.set_annotations_for_book(book_id, fmt, annots_list, user_type, user) 2516 2517 @write_api 2518 def merge_annotations_for_book(self, book_id, fmt, annots_list, user_type='local', user='viewer'): 2519 from calibre.utils.date import EPOCH 2520 from calibre.utils.iso8601 import parse_iso8601 2521 amap = self._annotations_map_for_book(book_id, fmt, user_type=user_type, user=user) 2522 merge_annotations(annots_list, amap) 2523 alist = [] 2524 for val in itervalues(amap): 2525 for annot in val: 2526 ts = (parse_iso8601(annot['timestamp']) - EPOCH).total_seconds() 2527 alist.append((annot, ts)) 2528 self._set_annotations_for_book(book_id, fmt, alist, user_type=user_type, user=user) 2529 2530 @write_api 2531 def reindex_annotations(self): 2532 self.backend.reindex_annotations() 2533 2534 2535def import_library(library_key, importer, library_path, progress=None, abort=None): 2536 from calibre.db.backend import DB 2537 metadata = importer.metadata[library_key] 2538 total = metadata['total'] 2539 if progress is not None: 2540 progress('metadata.db', 0, total) 2541 if abort is not None and abort.is_set(): 2542 return 2543 with open(os.path.join(library_path, 'metadata.db'), 'wb') as f: 2544 src = importer.start_file(metadata['metadata.db'], 'metadata.db for ' + library_path) 2545 shutil.copyfileobj(src, f) 2546 src.close() 2547 cache = Cache(DB(library_path, load_user_formatter_functions=False)) 2548 cache.init() 2549 format_data = {int(book_id):data for book_id, data in iteritems(metadata['format_data'])} 2550 for i, (book_id, fmt_key_map) in enumerate(iteritems(format_data)): 2551 if abort is not None and abort.is_set(): 2552 return 2553 title = cache._field_for('title', book_id) 2554 if progress is not None: 2555 progress(title, i + 1, total) 2556 cache._update_path((book_id,), mark_as_dirtied=False) 2557 for fmt, fmtkey in iteritems(fmt_key_map): 2558 if fmt == '.cover': 2559 stream = importer.start_file(fmtkey, _('Cover for %s') % title) 2560 path = cache._field_for('path', book_id).replace('/', os.sep) 2561 cache.backend.set_cover(book_id, path, stream, no_processing=True) 2562 else: 2563 stream = importer.start_file(fmtkey, _('{0} format for {1}').format(fmt.upper(), title)) 2564 size, fname = cache._do_add_format(book_id, fmt, stream, mtime=stream.mtime) 2565 cache.fields['formats'].table.update_fmt(book_id, fmt, fname, size, cache.backend) 2566 stream.close() 2567 cache.dump_metadata({book_id}) 2568 if progress is not None: 2569 progress(_('Completed'), total, total) 2570 return cache 2571# }}} 2572