1# -*- coding: utf-8 -*- 2 3 4__license__ = 'GPL v3' 5__copyright__ = '2009, John Schember <john at nachtimwald.com>' 6__docformat__ = 'restructuredtext en' 7 8''' 9Generic USB Mass storage device driver. This is not a complete stand alone 10driver. It is intended to be subclassed with the relevant parts implemented 11for a particular device. 12''' 13 14import os, time, json, shutil 15from itertools import cycle 16 17from calibre.constants import numeric_version, ismacos 18from calibre import prints, isbytestring, fsync 19from calibre.constants import filesystem_encoding, DEBUG 20from calibre.devices.usbms.cli import CLI 21from calibre.devices.usbms.device import Device 22from calibre.devices.usbms.books import BookList, Book 23from calibre.ebooks.metadata.book.json_codec import JsonCodec 24from polyglot.builtins import itervalues, string_or_bytes 25 26 27def debug_print(*args, **kw): 28 base_time = getattr(debug_print, 'base_time', None) 29 if base_time is None: 30 debug_print.base_time = base_time = time.monotonic() 31 if DEBUG: 32 prints('DEBUG: %6.1f'%(time.monotonic()-base_time), *args, **kw) 33 34 35def safe_walk(top, topdown=True, onerror=None, followlinks=False, maxdepth=128): 36 ' A replacement for os.walk that does not die when it encounters undecodeable filenames in a linux filesystem' 37 if maxdepth < 0: 38 return 39 islink, join, isdir = os.path.islink, os.path.join, os.path.isdir 40 41 # We may not have read permission for top, in which case we can't 42 # get a list of the files the directory contains. os.path.walk 43 # always suppressed the exception then, rather than blow up for a 44 # minor reason when (say) a thousand readable directories are still 45 # left to visit. That logic is copied here. 46 try: 47 names = os.listdir(top) 48 except OSError as err: 49 if onerror is not None: 50 onerror(err) 51 return 52 53 dirs, nondirs = [], [] 54 for name in names: 55 if isinstance(name, bytes): 56 try: 57 name = name.decode(filesystem_encoding) 58 except UnicodeDecodeError: 59 debug_print('Skipping undecodeable file: %r' % name) 60 continue 61 if isdir(join(top, name)): 62 dirs.append(name) 63 else: 64 nondirs.append(name) 65 66 if topdown: 67 yield top, dirs, nondirs 68 for name in dirs: 69 new_path = join(top, name) 70 if followlinks or not islink(new_path): 71 yield from safe_walk(new_path, topdown, onerror, followlinks, maxdepth-1) 72 if not topdown: 73 yield top, dirs, nondirs 74 75 76# CLI must come before Device as it implements the CLI functions that 77# are inherited from the device interface in Device. 78class USBMS(CLI, Device): 79 80 ''' 81 The base class for all USBMS devices. Implements the logic for 82 sending/getting/updating metadata/caching metadata/etc. 83 ''' 84 85 description = _('Communicate with an e-book reader.') 86 author = 'John Schember' 87 supported_platforms = ['windows', 'osx', 'linux'] 88 89 # Store type instances of BookList and Book. We must do this because 90 # a) we need to override these classes in some device drivers, and 91 # b) the classmethods seem only to see real attributes declared in the 92 # class, not attributes stored in the class 93 booklist_class = BookList 94 book_class = Book 95 96 FORMATS = [] 97 CAN_SET_METADATA = [] 98 METADATA_CACHE = 'metadata.calibre' 99 DRIVEINFO = 'driveinfo.calibre' 100 101 SCAN_FROM_ROOT = False 102 103 def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): 104 from calibre.utils.date import now, isoformat 105 import uuid 106 if not isinstance(dinfo, dict): 107 dinfo = {} 108 if dinfo.get('device_store_uuid', None) is None: 109 dinfo['device_store_uuid'] = str(uuid.uuid4()) 110 if dinfo.get('device_name', None) is None: 111 dinfo['device_name'] = self.get_gui_name() 112 if name is not None: 113 dinfo['device_name'] = name 114 dinfo['location_code'] = location_code 115 dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None) 116 dinfo['calibre_version'] = '.'.join([str(i) for i in numeric_version]) 117 dinfo['date_last_connected'] = isoformat(now()) 118 dinfo['prefix'] = prefix.replace('\\', '/') 119 return dinfo 120 121 def _update_driveinfo_file(self, prefix, location_code, name=None): 122 from calibre.utils.config import from_json, to_json 123 if os.path.exists(os.path.join(prefix, self.DRIVEINFO)): 124 with lopen(os.path.join(prefix, self.DRIVEINFO), 'rb') as f: 125 try: 126 driveinfo = json.loads(f.read(), object_hook=from_json) 127 except: 128 driveinfo = None 129 driveinfo = self._update_driveinfo_record(driveinfo, prefix, 130 location_code, name) 131 data = json.dumps(driveinfo, default=to_json) 132 if not isinstance(data, bytes): 133 data = data.encode('utf-8') 134 with lopen(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: 135 f.write(data) 136 fsync(f) 137 else: 138 driveinfo = self._update_driveinfo_record({}, prefix, location_code, name) 139 data = json.dumps(driveinfo, default=to_json) 140 if not isinstance(data, bytes): 141 data = data.encode('utf-8') 142 with lopen(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: 143 f.write(data) 144 fsync(f) 145 return driveinfo 146 147 def get_device_information(self, end_session=True): 148 self.report_progress(1.0, _('Get device information...')) 149 self.driveinfo = {} 150 151 def raise_os_error(e): 152 raise OSError(_('Failed to access files in the main memory of' 153 ' your device. You should contact the device' 154 ' manufacturer for support. Common fixes are:' 155 ' try a different USB cable/USB port on your computer.' 156 ' If you device has a "Reset to factory defaults" type' 157 ' of setting somewhere, use it. Underlying error: %s') 158 % e) from e 159 160 if self._main_prefix is not None: 161 try: 162 self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix, 'main') 163 except PermissionError as e: 164 if ismacos: 165 raise PermissionError(_( 166 'Permission was denied by macOS trying to access files in the main memory of' 167 ' your device. You will need to grant permission explicitly by looking under' 168 ' System Preferences > Security and Privacy > Privacy > Files and Folders.' 169 ' Underlying error: %s' 170 ) % e) from e 171 raise_os_error(e) 172 except OSError as e: 173 raise_os_error(e) 174 try: 175 if self._card_a_prefix is not None: 176 self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix, 'A') 177 if self._card_b_prefix is not None: 178 self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix, 'B') 179 except OSError as e: 180 raise OSError(_('Failed to access files on the SD card in your' 181 ' device. This can happen for many reasons. The SD card may be' 182 ' corrupted, it may be too large for your device, it may be' 183 ' write-protected, etc. Try a different SD card, or reformat' 184 ' your SD card using the FAT32 filesystem. Also make sure' 185 ' there are not too many files in the root of your SD card.' 186 ' Underlying error: %s') % e) 187 return (self.get_gui_name(), '', '', '', self.driveinfo) 188 189 def set_driveinfo_name(self, location_code, name): 190 if location_code == 'main': 191 self._update_driveinfo_file(self._main_prefix, location_code, name) 192 elif location_code == 'A': 193 self._update_driveinfo_file(self._card_a_prefix, location_code, name) 194 elif location_code == 'B': 195 self._update_driveinfo_file(self._card_b_prefix, location_code, name) 196 197 def formats_to_scan_for(self): 198 return set(self.settings().format_map) | set(self.FORMATS) 199 200 def is_allowed_book_file(self, filename, path, prefix): 201 return True 202 203 def books(self, oncard=None, end_session=True): 204 from calibre.ebooks.metadata.meta import path_to_ext 205 206 debug_print('USBMS: Fetching list of books from device. Device=', 207 self.__class__.__name__, 208 'oncard=', oncard) 209 210 dummy_bl = self.booklist_class(None, None, None) 211 212 if oncard == 'carda' and not self._card_a_prefix: 213 self.report_progress(1.0, _('Getting list of books on device...')) 214 return dummy_bl 215 elif oncard == 'cardb' and not self._card_b_prefix: 216 self.report_progress(1.0, _('Getting list of books on device...')) 217 return dummy_bl 218 elif oncard and oncard != 'carda' and oncard != 'cardb': 219 self.report_progress(1.0, _('Getting list of books on device...')) 220 return dummy_bl 221 222 prefix = self._card_a_prefix if oncard == 'carda' else \ 223 self._card_b_prefix if oncard == 'cardb' \ 224 else self._main_prefix 225 226 ebook_dirs = self.get_carda_ebook_dir() if oncard == 'carda' else \ 227 self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ 228 self.get_main_ebook_dir() 229 230 debug_print('USBMS: dirs are:', prefix, ebook_dirs) 231 232 # get the metadata cache 233 bl = self.booklist_class(oncard, prefix, self.settings) 234 need_sync = self.parse_metadata_cache(bl, prefix, self.METADATA_CACHE) 235 236 # make a dict cache of paths so the lookup in the loop below is faster. 237 bl_cache = {} 238 for idx, b in enumerate(bl): 239 bl_cache[b.lpath] = idx 240 241 all_formats = self.formats_to_scan_for() 242 243 def update_booklist(filename, path, prefix): 244 changed = False 245 if path_to_ext(filename) in all_formats and self.is_allowed_book_file(filename, path, prefix): 246 try: 247 lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] 248 if lpath.startswith(os.sep): 249 lpath = lpath[len(os.sep):] 250 lpath = lpath.replace('\\', '/') 251 idx = bl_cache.get(lpath, None) 252 if idx is not None: 253 bl_cache[lpath] = None 254 if self.update_metadata_item(bl[idx]): 255 # print 'update_metadata_item returned true' 256 changed = True 257 else: 258 if bl.add_book(self.book_from_path(prefix, lpath), 259 replace_metadata=False): 260 changed = True 261 except: # Probably a filename encoding error 262 import traceback 263 traceback.print_exc() 264 return changed 265 if isinstance(ebook_dirs, string_or_bytes): 266 ebook_dirs = [ebook_dirs] 267 for ebook_dir in ebook_dirs: 268 ebook_dir = self.path_to_unicode(ebook_dir) 269 if self.SCAN_FROM_ROOT: 270 ebook_dir = self.normalize_path(prefix) 271 else: 272 ebook_dir = self.normalize_path( 273 os.path.join(prefix, *(ebook_dir.split('/'))) 274 if ebook_dir else prefix) 275 debug_print('USBMS: scan from root', self.SCAN_FROM_ROOT, ebook_dir) 276 if not os.path.exists(ebook_dir): 277 continue 278 # Get all books in the ebook_dir directory 279 if self.SUPPORTS_SUB_DIRS or self.SUPPORTS_SUB_DIRS_FOR_SCAN: 280 # build a list of files to check, so we can accurately report progress 281 flist = [] 282 for path, dirs, files in safe_walk(ebook_dir): 283 for filename in files: 284 if filename != self.METADATA_CACHE: 285 flist.append({'filename': self.path_to_unicode(filename), 286 'path':self.path_to_unicode(path)}) 287 for i, f in enumerate(flist): 288 self.report_progress(i/float(len(flist)), _('Getting list of books on device...')) 289 changed = update_booklist(f['filename'], f['path'], prefix) 290 if changed: 291 need_sync = True 292 else: 293 paths = os.listdir(ebook_dir) 294 for i, filename in enumerate(paths): 295 self.report_progress((i+1) / float(len(paths)), _('Getting list of books on device...')) 296 changed = update_booklist(self.path_to_unicode(filename), ebook_dir, prefix) 297 if changed: 298 need_sync = True 299 300 # Remove books that are no longer in the filesystem. Cache contains 301 # indices into the booklist if book not in filesystem, None otherwise 302 # Do the operation in reverse order so indices remain valid 303 for idx in sorted(itervalues(bl_cache), reverse=True, key=lambda x: -1 if x is None else x): 304 if idx is not None: 305 need_sync = True 306 del bl[idx] 307 308 debug_print('USBMS: count found in cache: %d, count of files in metadata: %d, need_sync: %s' % 309 (len(bl_cache), len(bl), need_sync)) 310 if need_sync: # self.count_found_in_bl != len(bl) or need_sync: 311 if oncard == 'cardb': 312 self.sync_booklists((None, None, bl)) 313 elif oncard == 'carda': 314 self.sync_booklists((None, bl, None)) 315 else: 316 self.sync_booklists((bl, None, None)) 317 318 self.report_progress(1.0, _('Getting list of books on device...')) 319 debug_print('USBMS: Finished fetching list of books from device. oncard=', oncard) 320 return bl 321 322 def upload_books(self, files, names, on_card=None, end_session=True, 323 metadata=None): 324 debug_print('USBMS: uploading %d books'%(len(files))) 325 326 path = self._sanity_check(on_card, files) 327 328 paths = [] 329 names = iter(names) 330 metadata = iter(metadata) 331 332 for i, infile in enumerate(files): 333 mdata, fname = next(metadata), next(names) 334 filepath = self.normalize_path(self.create_upload_path(path, mdata, fname)) 335 if not hasattr(infile, 'read'): 336 infile = self.normalize_path(infile) 337 filepath = self.put_file(infile, filepath, replace_file=True) 338 paths.append(filepath) 339 try: 340 self.upload_cover(os.path.dirname(filepath), 341 os.path.splitext(os.path.basename(filepath))[0], 342 mdata, filepath) 343 except: # Failure to upload cover is not catastrophic 344 import traceback 345 traceback.print_exc() 346 347 self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) 348 349 self.report_progress(1.0, _('Transferring books to device...')) 350 debug_print('USBMS: finished uploading %d books'%(len(files))) 351 return list(zip(paths, cycle([on_card]))) 352 353 def upload_cover(self, path, filename, metadata, filepath): 354 ''' 355 Upload book cover to the device. Default implementation does nothing. 356 357 :param path: The full path to the folder where the associated book is located. 358 :param filename: The name of the book file without the extension. 359 :param metadata: metadata belonging to the book. Use metadata.thumbnail 360 for cover 361 :param filepath: The full path to the e-book file 362 363 ''' 364 pass 365 366 def add_books_to_metadata(self, locations, metadata, booklists): 367 debug_print('USBMS: adding metadata for %d books'%(len(metadata))) 368 369 metadata = iter(metadata) 370 locations = tuple(locations) 371 for i, location in enumerate(locations): 372 self.report_progress((i+1) / float(len(locations)), _('Adding books to device metadata listing...')) 373 info = next(metadata) 374 blist = 2 if location[1] == 'cardb' else 1 if location[1] == 'carda' else 0 375 376 # Extract the correct prefix from the pathname. To do this correctly, 377 # we must ensure that both the prefix and the path are normalized 378 # so that the comparison will work. Book's __init__ will fix up 379 # lpath, so we don't need to worry about that here. 380 path = self.normalize_path(location[0]) 381 if self._main_prefix: 382 prefix = self._main_prefix if \ 383 path.startswith(self.normalize_path(self._main_prefix)) else None 384 if not prefix and self._card_a_prefix: 385 prefix = self._card_a_prefix if \ 386 path.startswith(self.normalize_path(self._card_a_prefix)) else None 387 if not prefix and self._card_b_prefix: 388 prefix = self._card_b_prefix if \ 389 path.startswith(self.normalize_path(self._card_b_prefix)) else None 390 if prefix is None: 391 prints('in add_books_to_metadata. Prefix is None!', path, 392 self._main_prefix) 393 continue 394 lpath = path.partition(prefix)[2] 395 if lpath.startswith('/') or lpath.startswith('\\'): 396 lpath = lpath[1:] 397 book = self.book_class(prefix, lpath, other=info) 398 if book.size is None: 399 book.size = os.stat(self.normalize_path(path)).st_size 400 b = booklists[blist].add_book(book, replace_metadata=True) 401 if b: 402 b._new_book = True 403 self.report_progress(1.0, _('Adding books to device metadata listing...')) 404 debug_print('USBMS: finished adding metadata') 405 406 def delete_single_book(self, path): 407 os.unlink(path) 408 409 def delete_extra_book_files(self, path): 410 filepath = os.path.splitext(path)[0] 411 for ext in self.DELETE_EXTS: 412 for x in (filepath, path): 413 x += ext 414 if os.path.exists(x): 415 if os.path.isdir(x): 416 shutil.rmtree(x, ignore_errors=True) 417 else: 418 os.unlink(x) 419 420 if self.SUPPORTS_SUB_DIRS: 421 try: 422 os.removedirs(os.path.dirname(path)) 423 except: 424 pass 425 426 def delete_books(self, paths, end_session=True): 427 debug_print('USBMS: deleting %d books'%(len(paths))) 428 for i, path in enumerate(paths): 429 self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) 430 path = self.normalize_path(path) 431 if os.path.exists(path): 432 # Delete the ebook 433 self.delete_single_book(path) 434 self.delete_extra_book_files(path) 435 436 self.report_progress(1.0, _('Removing books from device...')) 437 debug_print('USBMS: finished deleting %d books'%(len(paths))) 438 439 def remove_books_from_metadata(self, paths, booklists): 440 debug_print('USBMS: removing metadata for %d books'%(len(paths))) 441 442 for i, path in enumerate(paths): 443 self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...')) 444 for bl in booklists: 445 for book in bl: 446 if path.endswith(book.path): 447 bl.remove_book(book) 448 self.report_progress(1.0, _('Removing books from device metadata listing...')) 449 debug_print('USBMS: finished removing metadata for %d books'%(len(paths))) 450 451 # If you override this method and you use book._new_book, then you must 452 # complete the processing before you call this method. The flag is cleared 453 # at the end just before the return 454 def sync_booklists(self, booklists, end_session=True): 455 debug_print('USBMS: starting sync_booklists') 456 json_codec = JsonCodec() 457 458 if not os.path.exists(self.normalize_path(self._main_prefix)): 459 os.makedirs(self.normalize_path(self._main_prefix)) 460 461 def write_prefix(prefix, listid): 462 if (prefix is not None and len(booklists) > listid and 463 isinstance(booklists[listid], self.booklist_class)): 464 if not os.path.exists(prefix): 465 os.makedirs(self.normalize_path(prefix)) 466 with lopen(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: 467 json_codec.encode_to_file(f, booklists[listid]) 468 fsync(f) 469 write_prefix(self._main_prefix, 0) 470 write_prefix(self._card_a_prefix, 1) 471 write_prefix(self._card_b_prefix, 2) 472 473 # Clear the _new_book indication, as we are supposed to be done with 474 # adding books at this point 475 for blist in booklists: 476 if blist is not None: 477 for book in blist: 478 book._new_book = False 479 480 self.report_progress(1.0, _('Sending metadata to device...')) 481 debug_print('USBMS: finished sync_booklists') 482 483 @classmethod 484 def build_template_regexp(cls): 485 from calibre.devices.utils import build_template_regexp 486 return build_template_regexp(cls.save_template()) 487 488 @classmethod 489 def path_to_unicode(cls, path): 490 if isbytestring(path): 491 path = path.decode(filesystem_encoding) 492 return path 493 494 @classmethod 495 def normalize_path(cls, path): 496 'Return path with platform native path separators' 497 if path is None: 498 return None 499 if os.sep == '\\': 500 path = path.replace('/', '\\') 501 else: 502 path = path.replace('\\', '/') 503 return cls.path_to_unicode(path) 504 505 @classmethod 506 def parse_metadata_cache(cls, bl, prefix, name): 507 json_codec = JsonCodec() 508 need_sync = False 509 cache_file = cls.normalize_path(os.path.join(prefix, name)) 510 if os.access(cache_file, os.R_OK): 511 try: 512 with lopen(cache_file, 'rb') as f: 513 json_codec.decode_from_file(f, bl, cls.book_class, prefix) 514 except: 515 import traceback 516 traceback.print_exc() 517 bl = [] 518 need_sync = True 519 else: 520 need_sync = True 521 return need_sync 522 523 @classmethod 524 def update_metadata_item(cls, book): 525 changed = False 526 size = os.stat(cls.normalize_path(book.path)).st_size 527 if size != book.size: 528 changed = True 529 mi = cls.metadata_from_path(book.path) 530 book.smart_update(mi) 531 book.size = size 532 return changed 533 534 @classmethod 535 def metadata_from_path(cls, path): 536 return cls.metadata_from_formats([path]) 537 538 @classmethod 539 def metadata_from_formats(cls, fmts): 540 from calibre.ebooks.metadata.meta import metadata_from_formats 541 from calibre.customize.ui import quick_metadata 542 with quick_metadata: 543 return metadata_from_formats(fmts, force_read_metadata=True, 544 pattern=cls.build_template_regexp()) 545 546 @classmethod 547 def book_from_path(cls, prefix, lpath): 548 from calibre.ebooks.metadata.book.base import Metadata 549 550 if cls.settings().read_metadata or cls.MUST_READ_METADATA: 551 mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath))) 552 else: 553 from calibre.ebooks.metadata.meta import metadata_from_filename 554 mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)), 555 cls.build_template_regexp()) 556 if mi is None: 557 mi = Metadata(os.path.splitext(os.path.basename(lpath))[0], 558 [_('Unknown')]) 559 size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size 560 book = cls.book_class(prefix, lpath, other=mi, size=size) 561 return book 562