1# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*- 2# 3# Copyright 2015 Yigal Asnis 4# 5# This file is free software; you can redistribute it and/or modify it 6# under the terms of the GNU General Public License as published by the 7# Free Software Foundation; either version 2 of the License, or (at your 8# option) any later version. 9# 10# It is distributed in the hope that it will be useful, but 11# WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with duplicity; if not, write to the Free Software Foundation, 17# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 19from builtins import next 20from builtins import str 21 22import os 23 24from duplicity import log 25from duplicity import util 26from duplicity.errors import BackendException 27import duplicity.backend 28 29 30class PyDriveBackend(duplicity.backend.Backend): 31 u"""Connect to remote store using PyDrive API""" 32 33 def __init__(self, parsed_url): 34 duplicity.backend.Backend.__init__(self, parsed_url) 35 try: 36 import httplib2 37 from apiclient.discovery import build 38 except ImportError as e: 39 raise BackendException(u"""\ 40PyDrive backend requires PyDrive and Google API client installation. 41Please read the manpage for setup details. 42Exception: %s""" % str(e)) 43 44 # Shared Drive ID specified as a query parameter in the backend URL. 45 # Example: pydrive://developer.gserviceaccount.com/target-folder/?driveID=<SHARED DRIVE ID> 46 self.api_params = {} 47 self.shared_drive_id = None 48 if u'driveID' in parsed_url.query_args: 49 self.shared_drive_id = parsed_url.query_args[u'driveID'][0] 50 self.api_params = {u'corpora': u'teamDrive', 51 u'teamDriveId': self.shared_drive_id, 52 u'includeTeamDriveItems': True, 53 u'supportsTeamDrives': True} 54 55 try: 56 from pydrive2.auth import GoogleAuth 57 from pydrive2.drive import GoogleDrive 58 from pydrive2.files import ApiRequestError, FileNotUploadedError 59 except ImportError as e: 60 try: 61 from pydrive.auth import GoogleAuth 62 from pydrive.drive import GoogleDrive 63 from pydrive.files import ApiRequestError, FileNotUploadedError 64 except ImportError as e: 65 raise BackendException(u"""\ 66PyDrive backend requires PyDrive installation. Please read the manpage for setup details. 67Exception: %s""" % str(e)) 68 69 # let user get by with old client while he can 70 try: 71 from oauth2client.client import SignedJwtAssertionCredentials 72 self.oldClient = True 73 except: 74 from oauth2client.service_account import ServiceAccountCredentials 75 from oauth2client import crypt 76 self.oldClient = False 77 78 if u'GOOGLE_DRIVE_ACCOUNT_KEY' in os.environ: 79 account_key = os.environ[u'GOOGLE_DRIVE_ACCOUNT_KEY'] 80 if self.oldClient: 81 credentials = SignedJwtAssertionCredentials(parsed_url.username + 82 u'@' + parsed_url.hostname, 83 account_key, 84 scopes=u'https://www.googleapis.com/auth/drive') 85 else: 86 signer = crypt.Signer.from_string(account_key) 87 credentials = ServiceAccountCredentials(parsed_url.username + u'@' + parsed_url.hostname, signer, 88 scopes=u'https://www.googleapis.com/auth/drive') 89 credentials.authorize(httplib2.Http()) 90 gauth = GoogleAuth(http_timeout=60) 91 gauth.credentials = credentials 92 elif u'GOOGLE_DRIVE_SETTINGS' in os.environ: 93 gauth = GoogleAuth(settings_file=os.environ[u'GOOGLE_DRIVE_SETTINGS'], http_timeout=60) 94 gauth.CommandLineAuth() 95 elif (u'GOOGLE_SECRETS_FILE' in os.environ and u'GOOGLE_CREDENTIALS_FILE' in os.environ): 96 gauth = GoogleAuth(http_timeout=60) 97 gauth.LoadClientConfigFile(os.environ[u'GOOGLE_SECRETS_FILE']) 98 gauth.LoadCredentialsFile(os.environ[u'GOOGLE_CREDENTIALS_FILE']) 99 if gauth.credentials is None: 100 gauth.CommandLineAuth() 101 elif gauth.access_token_expired: 102 gauth.Refresh() 103 else: 104 gauth.Authorize() 105 gauth.SaveCredentialsFile(os.environ[u'GOOGLE_CREDENTIALS_FILE']) 106 else: 107 raise BackendException( 108 u'GOOGLE_DRIVE_ACCOUNT_KEY or GOOGLE_DRIVE_SETTINGS environment ' 109 u'variable not set. Please read the manpage to fix.') 110 self.drive = GoogleDrive(gauth) 111 112 if self.shared_drive_id: 113 parent_folder_id = self.shared_drive_id 114 else: 115 # Dirty way to find root folder id 116 file_list = self.drive.ListFile({u'q': u"'Root' in parents and trashed=false"}).GetList() 117 if file_list: 118 parent_folder_id = file_list[0][u'parents'][0][u'id'] 119 else: 120 file_in_root = self.drive.CreateFile({u'title': u'i_am_in_root'}) 121 file_in_root.Upload() 122 parent_folder_id = file_in_root[u'parents'][0][u'id'] 123 file_in_root.Delete() 124 125 # Fetch destination folder entry and create hierarchy if required. 126 folder_names = parsed_url.path.split(u'/') 127 for folder_name in folder_names: 128 if not folder_name: 129 continue 130 list_file_args = {u'q': u"'" + parent_folder_id + 131 u"' in parents and trashed=false"} 132 list_file_args.update(self.api_params) 133 file_list = self.drive.ListFile(list_file_args).GetList() 134 folder = next((item for item in file_list if item[u'title'] == folder_name and 135 item[u'mimeType'] == u'application/vnd.google-apps.folder'), None) 136 if folder is None: 137 create_file_args = {u'title': folder_name, 138 u'mimeType': u"application/vnd.google-apps.folder", 139 u'parents': [{u'id': parent_folder_id}]} 140 create_file_args[u'parents'][0].update(self.api_params) 141 create_file_args.update(self.api_params) 142 folder = self.drive.CreateFile(create_file_args) 143 if self.shared_drive_id: 144 folder.Upload(param={u'supportsTeamDrives': True}) 145 else: 146 folder.Upload() 147 parent_folder_id = folder[u'id'] 148 self.folder = parent_folder_id 149 self.id_cache = {} 150 151 def file_by_name(self, filename): 152 try: 153 from pydrive2.files import ApiRequestError # pylint: disable=import-error 154 except ImportError: 155 from pydrive.files import ApiRequestError # pylint: disable=import-error 156 157 filename = util.fsdecode(filename) # PyDrive deals with unicode filenames 158 159 if filename in self.id_cache: 160 # It might since have been locally moved, renamed or deleted, so we 161 # need to validate the entry. 162 file_id = self.id_cache[filename] 163 drive_file = self.drive.CreateFile({u'id': file_id}) 164 try: 165 if drive_file[u'title'] == filename and not drive_file[u'labels'][u'trashed']: 166 for parent in drive_file[u'parents']: 167 if parent[u'id'] == self.folder: 168 log.Info(u"PyDrive backend: found file '%s' with id %s in ID cache" % 169 (filename, file_id)) 170 return drive_file 171 except ApiRequestError as error: 172 # A 404 occurs if the ID is no longer valid 173 if error.args[0].resp.status != 404: 174 raise 175 # If we get here, the cache entry is invalid 176 log.Info(u"PyDrive backend: invalidating '%s' (previously ID %s) from ID cache" % 177 (filename, file_id)) 178 del self.id_cache[filename] 179 180 # Not found in the cache, so use directory listing. This is less 181 # reliable because there is no strong consistency. 182 q = u"title='%s' and '%s' in parents and trashed=false" % (filename, self.folder) 183 fields = u'items(title,id,fileSize,downloadUrl,exportLinks),nextPageToken' 184 list_file_args = {u'q': q, u'fields': fields} 185 list_file_args.update(self.api_params) 186 flist = self.drive.ListFile(list_file_args).GetList() 187 if len(flist) > 1: 188 log.FatalError(_(u"PyDrive backend: multiple files called '%s'.") % (filename,)) 189 elif flist: 190 file_id = flist[0][u'id'] 191 self.id_cache[filename] = flist[0][u'id'] 192 log.Info(u"PyDrive backend: found file '%s' with id %s on server, " 193 u"adding to cache" % (filename, file_id)) 194 return flist[0] 195 log.Info(u"PyDrive backend: file '%s' not found in cache or on server" % 196 (filename,)) 197 return None 198 199 def id_by_name(self, filename): 200 drive_file = self.file_by_name(filename) 201 if drive_file is None: 202 return u'' 203 else: 204 return drive_file[u'id'] 205 206 def _put(self, source_path, remote_filename): 207 remote_filename = util.fsdecode(remote_filename) 208 drive_file = self.file_by_name(remote_filename) 209 if drive_file is None: 210 # No existing file, make a new one 211 create_file_args = {u'title': remote_filename, 212 u'parents': [{u"kind": u"drive#fileLink", 213 u"id": self.folder}]} 214 create_file_args[u'parents'][0].update(self.api_params) 215 drive_file = self.drive.CreateFile(create_file_args) 216 log.Info(u"PyDrive backend: creating new file '%s'" % (remote_filename,)) 217 else: 218 log.Info(u"PyDrive backend: replacing existing file '%s' with id '%s'" % ( 219 remote_filename, drive_file[u'id'])) 220 drive_file.SetContentFile(util.fsdecode(source_path.name)) 221 if self.shared_drive_id: 222 drive_file.Upload(param={u'supportsTeamDrives': True}) 223 else: 224 drive_file.Upload() 225 self.id_cache[remote_filename] = drive_file[u'id'] 226 227 def _get(self, remote_filename, local_path): 228 drive_file = self.file_by_name(remote_filename) 229 drive_file.GetContentFile(util.fsdecode(local_path.name)) 230 231 def _list(self): 232 list_file_args = { 233 u'q': u"'" + self.folder + u"' in parents and trashed=false", 234 u'fields': u'items(title,id),nextPageToken'} 235 list_file_args.update(self.api_params) 236 drive_files = self.drive.ListFile(list_file_args).GetList() 237 filenames = set(item[u'title'] for item in drive_files) 238 # Check the cache as well. A file might have just been uploaded but 239 # not yet appear in the listing. 240 # Note: do not use iterkeys() here, because file_by_name will modify 241 # the cache if it finds invalid entries. 242 for filename in list(self.id_cache.keys()): 243 if (filename not in filenames) and (self.file_by_name(filename) is not None): 244 filenames.add(filename) 245 return list(filenames) 246 247 def _delete(self, filename): 248 file_id = self.id_by_name(filename) 249 if file_id == u'': 250 log.Warn(u"File '%s' does not exist while trying to delete it" % (util.fsdecode(filename),)) 251 elif self.shared_drive_id: 252 self.drive.auth.service.files().delete(fileId=file_id, param={u'supportsTeamDrives': True}).execute() 253 else: 254 self.drive.auth.service.files().delete(fileId=file_id).execute() 255 256 def _query(self, filename): 257 drive_file = self.file_by_name(filename) 258 if drive_file is None: 259 size = -1 260 else: 261 size = int(drive_file[u'fileSize']) 262 return {u'size': size} 263 264 def _error_code(self, operation, error): # pylint: disable=unused-argument 265 try: 266 from pydrive2.files import ApiRequestError, FileNotUploadedError # pylint: disable=import-error 267 except ImportError: 268 from pydrive.files import ApiRequestError, FileNotUploadedError # pylint: disable=import-error 269 if isinstance(error, FileNotUploadedError): 270 return log.ErrorCode.backend_not_found 271 elif isinstance(error, ApiRequestError): 272 return log.ErrorCode.backend_permission_denied 273 return log.ErrorCode.backend_error 274 275 276duplicity.backend.register_backend(u'pydrive', PyDriveBackend) 277u""" pydrive is an alternate way to access gdocs """ 278duplicity.backend.register_backend(u'pydrive+gdocs', PyDriveBackend) 279u""" register pydrive as the default way to access gdocs """ 280duplicity.backend.register_backend(u'gdocs', PyDriveBackend) 281 282duplicity.backend.uses_netloc.extend([u'pydrive', u'pydrive+gdocs', u'gdocs']) 283