1#!/usr/local/bin/python3.8 2# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai 3 4__license__ = 'GPL v3' 5__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>' 6__docformat__ = 'restructuredtext en' 7 8import operator, traceback, pprint, sys, time 9from threading import RLock 10from collections import namedtuple 11from functools import partial 12 13from calibre import prints, as_unicode, force_unicode 14from calibre.constants import islinux, ismacos 15from calibre.ptempfile import SpooledTemporaryFile 16from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice, OpenActionNeeded 17from calibre.devices.mtp.base import MTPDeviceBase, synchronous, debug 18 19MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' 20 'bcd serial manufacturer product') 21 22null = object() 23 24 25def fingerprint(d): 26 return MTPDevice(d.busnum, d.devnum, d.vendor_id, d.product_id, d.bcd, 27 d.serial, d.manufacturer, d.product) 28 29 30APPLE = 0x05ac 31 32 33class MTP_DEVICE(MTPDeviceBase): 34 35 supported_platforms = ['freebsd', 'linux', 'osx'] 36 37 def __init__(self, *args, **kwargs): 38 MTPDeviceBase.__init__(self, *args, **kwargs) 39 self.libmtp = None 40 self.known_devices = None 41 self.detect_cache = {} 42 43 self.dev = None 44 self._filesystem_cache = None 45 self.lock = RLock() 46 self.blacklisted_devices = set() 47 self.ejected_devices = set() 48 self.currently_connected_dev = None 49 self._is_device_mtp = None 50 if islinux: 51 from calibre.devices.mtp.unix.sysfs import MTPDetect 52 self._is_device_mtp = MTPDetect() 53 if ismacos and 'osx' in self.supported_platforms: 54 from calibre_extensions import usbobserver 55 self.usbobserver = usbobserver 56 self._is_device_mtp = self.osx_is_device_mtp 57 58 def is_device_mtp(self, d, debug=None): 59 ''' Returns True iff the _is_device_mtp check returns True and libmtp 60 is able to probe the device successfully. ''' 61 if self._is_device_mtp is None: 62 return False 63 return (self._is_device_mtp(d, debug=debug) and 64 self.libmtp.is_mtp_device(d.busnum, d.devnum)) 65 66 def osx_is_device_mtp(self, d, debug=None): 67 if not d.serial: 68 ans = False 69 else: 70 try: 71 ans = self.usbobserver.is_mtp_device(d.vendor_id, d.product_id, d.bcd, d.serial) 72 except Exception: 73 if debug is not None: 74 import traceback 75 traceback.print_stack() 76 return False 77 if debug is not None and ans: 78 debug('Device {} claims to be an MTP device in the IOKit registry'.format(d)) 79 return bool(ans) 80 81 def set_debug_level(self, lvl): 82 self.libmtp.set_debug_level(lvl) 83 84 @synchronous 85 def detect_managed_devices(self, devices_on_system, force_refresh=False): 86 if self.libmtp is None: 87 return None 88 # First remove blacklisted devices. 89 devs = set() 90 for d in devices_on_system: 91 fp = fingerprint(d) 92 if fp not in self.blacklisted_devices and fp.vendor_id != APPLE: 93 # Do not try to open Apple devices 94 devs.add(fp) 95 96 # Clean up ejected devices 97 self.ejected_devices = devs.intersection(self.ejected_devices) 98 99 # Check if the currently connected device is still present 100 if self.currently_connected_dev is not None: 101 return (self.currently_connected_dev if 102 self.currently_connected_dev in devs else None) 103 104 # Remove ejected devices 105 devs = devs - self.ejected_devices 106 107 # Now check for MTP devices 108 if force_refresh: 109 self.detect_cache = {} 110 cache = self.detect_cache 111 for d in devs: 112 ans = cache.get(d, None) 113 if ans is None: 114 ans = ( 115 (d.vendor_id, d.product_id) in self.known_devices or 116 self.is_device_mtp(d)) 117 cache[d] = ans 118 if ans: 119 return d 120 121 return None 122 123 @synchronous 124 def debug_managed_device_detection(self, devices_on_system, output): 125 if self.currently_connected_dev is not None: 126 return True 127 p = partial(prints, file=output) 128 if self.libmtp is None: 129 err = 'startup() not called on this device driver' 130 p(err) 131 return False 132 devs = [d for d in devices_on_system if 133 ((d.vendor_id, d.product_id) in self.known_devices or 134 self.is_device_mtp(d, debug=p)) and d.vendor_id != APPLE] 135 if not devs: 136 p('No MTP devices connected to system') 137 return False 138 p('MTP devices connected:') 139 for d in devs: 140 p(d) 141 142 for d in devs: 143 p('\nTrying to open:', d) 144 try: 145 self.open(d, 'debug') 146 except BlacklistedDevice: 147 p('This device has been blacklisted by the user') 148 continue 149 except: 150 p('Opening device failed:') 151 p(traceback.format_exc()) 152 return False 153 else: 154 p('Opened', self.current_friendly_name, 'successfully') 155 p('Storage info:') 156 p(pprint.pformat(self.dev.storage_info)) 157 self.post_yank_cleanup() 158 return True 159 return False 160 161 @synchronous 162 def create_device(self, connected_device): 163 d = connected_device 164 man, prod = d.manufacturer, d.product 165 man = force_unicode(man, 'utf-8') if isinstance(man, bytes) else man 166 prod = force_unicode(prod, 'utf-8') if isinstance(prod, bytes) else prod 167 return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id, 168 d.product_id, man, prod, d.serial) 169 170 @synchronous 171 def eject(self): 172 if self.currently_connected_dev is None: 173 return 174 self.ejected_devices.add(self.currently_connected_dev) 175 self.post_yank_cleanup() 176 177 @synchronous 178 def post_yank_cleanup(self): 179 self.dev = self._filesystem_cache = self.current_friendly_name = None 180 self.currently_connected_dev = None 181 self.current_serial_num = None 182 183 @property 184 def is_mtp_device_connected(self): 185 return self.currently_connected_dev is not None 186 187 @synchronous 188 def startup(self): 189 try: 190 from calibre_extensions import libmtp 191 except Exception as err: 192 print('Failed to load libmtp, MTP device detection disabled') 193 print(err) 194 self.libmtp = None 195 else: 196 self.libmtp = libmtp 197 self.known_devices = frozenset(self.libmtp.known_devices()) 198 199 for x in vars(self.libmtp): 200 if x.startswith('LIBMTP'): 201 setattr(self, x, getattr(self.libmtp, x)) 202 203 @synchronous 204 def shutdown(self): 205 self.dev = self._filesystem_cache = None 206 207 def format_errorstack(self, errs): 208 return '\n'.join('%d:%s'%(code, as_unicode(msg)) for code, msg in errs) 209 210 @synchronous 211 def open(self, connected_device, library_uuid): 212 self.dev = self._filesystem_cache = None 213 214 try: 215 self.dev = self.create_device(connected_device) 216 except Exception as e: 217 self.blacklisted_devices.add(connected_device) 218 raise OpenFailed('Failed to open %s: Error: %s'%( 219 connected_device, as_unicode(e))) 220 221 try: 222 storage = sorted(self.dev.storage_info, key=operator.itemgetter('id')) 223 except self.libmtp.MTPError as e: 224 if "The device has no storage information." in str(e): 225 # This happens on newer Android devices while waiting for 226 # the user to allow access. Apparently what happens is 227 # that when the user clicks allow, the device disconnects 228 # and re-connects as a new device. 229 name = self.dev.friendly_name or '' 230 if not name: 231 if connected_device.manufacturer: 232 name = connected_device.manufacturer 233 if connected_device.product: 234 name = name and (name + ' ') 235 name += connected_device.product 236 name = name or _('Unnamed device') 237 raise OpenActionNeeded(name, _( 238 'The device {0} is not allowing connections.' 239 ' Unlock the screen on the {0}, tap "Allow" on any connection popup message you see,' 240 ' then either wait a minute or restart calibre. You might' 241 ' also have to change the mode of the USB connection on the {0}' 242 ' to "Media Transfer mode (MTP)" or similar.' 243 ).format(name), (name, self.dev.serial_number)) 244 raise 245 246 storage = [x for x in storage if x.get('rw', False)] 247 if not storage: 248 self.blacklisted_devices.add(connected_device) 249 raise OpenFailed('No storage found for device %s'%(connected_device,)) 250 snum = self.dev.serial_number 251 if snum in self.prefs.get('blacklist', []): 252 self.blacklisted_devices.add(connected_device) 253 self.dev = None 254 raise BlacklistedDevice( 255 'The %s device has been blacklisted by the user'%(connected_device,)) 256 self._main_id = storage[0]['id'] 257 self._carda_id = self._cardb_id = None 258 if len(storage) > 1: 259 self._carda_id = storage[1]['id'] 260 if len(storage) > 2: 261 self._cardb_id = storage[2]['id'] 262 self.current_friendly_name = self.dev.friendly_name 263 if not self.current_friendly_name: 264 self.current_friendly_name = self.dev.model_name or _('Unknown MTP device') 265 self.current_serial_num = snum 266 self.currently_connected_dev = connected_device 267 268 @synchronous 269 def device_debug_info(self): 270 ans = self.get_gui_name() 271 ans += '\nSerial number: %s'%self.current_serial_num 272 ans += '\nManufacturer: %s'%self.dev.manufacturer_name 273 ans += '\nModel: %s'%self.dev.model_name 274 ans += '\nids: %s'%(self.dev.ids,) 275 ans += '\nDevice version: %s'%self.dev.device_version 276 ans += '\nStorage:\n' 277 storage = sorted(self.dev.storage_info, key=operator.itemgetter('id')) 278 ans += pprint.pformat(storage) 279 return ans 280 281 def _filesystem_callback(self, fs_map, entry, level): 282 name = entry.get('name', '') 283 self.filesystem_callback(_('Found object: %s')%name) 284 fs_map[entry.get('id', null)] = entry 285 path = [name] 286 pid = entry.get('parent_id', 0) 287 while pid != 0 and pid in fs_map: 288 parent = fs_map[pid] 289 path.append(parent.get('name', '')) 290 pid = parent.get('parent_id', 0) 291 if fs_map.get(pid, None) is parent: 292 break # An object is its own parent 293 path = tuple(reversed(path)) 294 ok = not self.is_folder_ignored(self._currently_getting_sid, path) 295 if not ok: 296 debug('Ignored object: %s' % '/'.join(path)) 297 return ok 298 299 @property 300 def filesystem_cache(self): 301 if self._filesystem_cache is None: 302 st = time.time() 303 debug('Loading filesystem metadata...') 304 from calibre.devices.mtp.filesystem_cache import FilesystemCache 305 with self.lock: 306 storage, all_items, all_errs = [], [], [] 307 for sid, capacity in zip([self._main_id, self._carda_id, 308 self._cardb_id], self.total_space()): 309 if sid is None: 310 continue 311 name = _('Unknown') 312 for x in self.dev.storage_info: 313 if x['id'] == sid: 314 name = x['name'] 315 break 316 storage.append({'id':sid, 'size':capacity, 317 'is_folder':True, 'name':name, 'can_delete':False, 318 'is_system':True}) 319 self._currently_getting_sid = str(sid) 320 items, errs = self.dev.get_filesystem(sid, 321 partial(self._filesystem_callback, {})) 322 all_items.extend(items), all_errs.extend(errs) 323 if not all_items and all_errs: 324 raise DeviceError( 325 'Failed to read filesystem from %s with errors: %s' 326 %(self.current_friendly_name, 327 self.format_errorstack(all_errs))) 328 if all_errs: 329 prints('There were some errors while getting the ' 330 ' filesystem from %s: %s'%( 331 self.current_friendly_name, 332 self.format_errorstack(all_errs))) 333 self._filesystem_cache = FilesystemCache(storage, all_items) 334 debug('Filesystem metadata loaded in %g seconds (%d objects)'%( 335 time.time()-st, len(self._filesystem_cache))) 336 return self._filesystem_cache 337 338 @synchronous 339 def get_basic_device_information(self): 340 d = self.dev 341 return (self.current_friendly_name, d.device_version, d.device_version, '') 342 343 @synchronous 344 def total_space(self, end_session=True): 345 ans = [0, 0, 0] 346 for s in self.dev.storage_info: 347 i = {self._main_id:0, self._carda_id:1, 348 self._cardb_id:2}.get(s['id'], None) 349 if i is not None: 350 ans[i] = s['capacity'] 351 return tuple(ans) 352 353 @synchronous 354 def free_space(self, end_session=True): 355 self.dev.update_storage_info() 356 ans = [0, 0, 0] 357 for s in self.dev.storage_info: 358 i = {self._main_id:0, self._carda_id:1, 359 self._cardb_id:2}.get(s['id'], None) 360 if i is not None: 361 ans[i] = s['freespace_bytes'] 362 return tuple(ans) 363 364 @synchronous 365 def create_folder(self, parent, name): 366 if not parent.is_folder: 367 raise ValueError('%s is not a folder'%(parent.full_path,)) 368 e = parent.folder_named(name) 369 if e is not None: 370 return e 371 sid, pid = parent.storage_id, parent.object_id 372 if pid == sid: 373 pid = 0 374 ans, errs = self.dev.create_folder(sid, pid, name) 375 if ans is None: 376 raise DeviceError( 377 'Failed to create folder named %s in %s with error: %s'% 378 (name, parent.full_path, self.format_errorstack(errs))) 379 return parent.add_child(ans) 380 381 @synchronous 382 def put_file(self, parent, name, stream, size, callback=None, replace=True): 383 e = parent.folder_named(name) 384 if e is not None: 385 raise ValueError('Cannot upload file, %s already has a folder named: %s'%( 386 parent.full_path, e.name)) 387 e = parent.file_named(name) 388 if e is not None: 389 if not replace: 390 raise ValueError('Cannot upload file %s, it already exists'%( 391 e.full_path,)) 392 self.delete_file_or_folder(e) 393 sid, pid = parent.storage_id, parent.object_id 394 if pid == sid: 395 pid = 0xFFFFFFFF 396 397 ans, errs = self.dev.put_file(sid, pid, name, stream, size, callback) 398 if ans is None: 399 raise DeviceError('Failed to upload file named: %s to %s: %s' 400 %(name, parent.full_path, self.format_errorstack(errs))) 401 return parent.add_child(ans) 402 403 @synchronous 404 def get_mtp_file(self, f, stream=None, callback=None): 405 if f.is_folder: 406 raise ValueError('%s if a folder'%(f.full_path,)) 407 set_name = stream is None 408 if stream is None: 409 stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat') 410 ok, errs = self.dev.get_file(f.object_id, stream, callback) 411 if not ok: 412 raise DeviceError('Failed to get file: %s with errors: %s'%( 413 f.full_path, self.format_errorstack(errs))) 414 stream.seek(0) 415 if set_name: 416 stream.name = f.name 417 return stream 418 419 @synchronous 420 def delete_file_or_folder(self, obj): 421 if obj.deleted: 422 return 423 if not obj.can_delete: 424 raise ValueError('Cannot delete %s as deletion not allowed'% 425 (obj.full_path,)) 426 if obj.is_system: 427 raise ValueError('Cannot delete %s as it is a system object'% 428 (obj.full_path,)) 429 if obj.files or obj.folders: 430 raise ValueError('Cannot delete %s as it is not empty'% 431 (obj.full_path,)) 432 parent = obj.parent 433 ok, errs = self.dev.delete_object(obj.object_id) 434 if not ok: 435 raise DeviceError('Failed to delete %s with error: %s'% 436 (obj.full_path, self.format_errorstack(errs))) 437 parent.remove_child(obj) 438 return parent 439 440 441def develop(): 442 from calibre.devices.scanner import DeviceScanner 443 scanner = DeviceScanner() 444 scanner.scan() 445 dev = MTP_DEVICE(None) 446 dev.startup() 447 try: 448 cd = dev.detect_managed_devices(scanner.devices) 449 if cd is None: 450 raise RuntimeError('No MTP device found') 451 dev.open(cd, 'develop') 452 pprint.pprint(dev.dev.storage_info) 453 dev.filesystem_cache 454 finally: 455 dev.shutdown() 456 457 458if __name__ == '__main__': 459 dev = MTP_DEVICE(None) 460 dev.startup() 461 from calibre.devices.scanner import DeviceScanner 462 scanner = DeviceScanner() 463 scanner.scan() 464 devs = scanner.devices 465 dev.debug_managed_device_detection(devs, sys.stdout) 466 dev.set_debug_level(dev.LIBMTP_DEBUG_ALL) 467 dev.shutdown() 468