1# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*- 2# 3# Copyright 2002 Ben Escoto <ben@emerose.org> 4# Copyright 2007 Kenneth Loafman <kenneth@loafman.com> 5# Copyright 2013 Edgar Soldin 6# - ssl cert verification, some robustness enhancements 7# 8# This file is part of duplicity. 9# 10# Duplicity is free software; you can redistribute it and/or modify it 11# under the terms of the GNU General Public License as published by the 12# Free Software Foundation; either version 2 of the License, or (at your 13# option) any later version. 14# 15# Duplicity is distributed in the hope that it will be useful, but 16# WITHOUT ANY WARRANTY; without even the implied warranty of 17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18# General Public License for more details. 19# 20# You should have received a copy of the GNU General Public License 21# along with duplicity; if not, write to the Free Software Foundation, 22# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23 24from future import standard_library 25standard_library.install_aliases() 26from builtins import str 27from builtins import range 28import base64 29import http.client 30import os 31import re 32import shutil 33import urllib.request # pylint: disable=import-error 34import urllib.parse # pylint: disable=import-error 35import urllib.error # pylint: disable=import-error 36import xml.dom.minidom 37 38import duplicity.backend 39from duplicity import config 40from duplicity import log 41from duplicity import util 42from duplicity.errors import BackendException, FatalBackendException 43 44 45class CustomMethodRequest(urllib.request.Request): 46 u""" 47 This request subclass allows explicit specification of 48 the HTTP request method. Basic urllib.request.Request class 49 chooses GET or POST depending on self.has_data() 50 """ 51 def __init__(self, method, *args, **kwargs): 52 self.method = method 53 urllib.request.Request.__init__(self, *args, **kwargs) 54 55 def get_method(self): 56 return self.method 57 58 59class VerifiedHTTPSConnection(http.client.HTTPSConnection): 60 def __init__(self, *args, **kwargs): 61 try: 62 global socket, ssl 63 import socket 64 import ssl 65 except ImportError: 66 raise FatalBackendException(_(u"Missing socket or ssl python modules.")) 67 68 http.client.HTTPSConnection.__init__(self, *args, **kwargs) 69 70 self.cacert_file = config.ssl_cacert_file 71 self.cacert_candidates = [u"~/.duplicity/cacert.pem", 72 u"~/duplicity_cacert.pem", 73 u"/etc/duplicity/cacert.pem"] 74 # if no cacert file was given search default locations 75 if not self.cacert_file: 76 for path in self.cacert_candidates: 77 path = os.path.expanduser(path) 78 if (os.path.isfile(path)): 79 self.cacert_file = path 80 break 81 82 # check if file is accessible (libssl errors are not very detailed) 83 if self.cacert_file and not os.access(self.cacert_file, os.R_OK): 84 raise FatalBackendException(_(u"Cacert database file '%s' is not readable.") % 85 self.cacert_file) 86 87 def connect(self): 88 # create new socket 89 sock = socket.create_connection((self.host, self.port), 90 self.timeout) 91 if self._tunnel_host: 92 self.sock = sock 93 self.tunnel() 94 95 # python 2.7.9+ supports default system certs now 96 if u"create_default_context" in dir(ssl): 97 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, 98 cafile=self.cacert_file, 99 capath=config.ssl_cacert_path) 100 self.sock = context.wrap_socket(sock, server_hostname=self.host) 101 # the legacy way needing a cert file 102 else: 103 if config.ssl_cacert_path: 104 raise FatalBackendException( 105 _(u"Option '--ssl-cacert-path' is not supported " 106 u"with python 2.7.8 and below.")) 107 108 if not self.cacert_file: 109 raise FatalBackendException(_(u"""\ 110For certificate verification with python 2.7.8 or earlier a cacert database 111file is needed in one of these locations: %s 112Hints: 113Consult the man page, chapter 'SSL Certificate Verification'. 114Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""") % 115 u", ".join(self.cacert_candidates)) 116 117 # wrap the socket in ssl using verification 118 self.sock = ssl.wrap_socket(sock, 119 cert_reqs=ssl.CERT_REQUIRED, 120 ca_certs=self.cacert_file, 121 ) 122 123 def request(self, *args, **kwargs): # pylint: disable=method-hidden 124 try: 125 return http.client.HTTPSConnection.request(self, *args, **kwargs) 126 except ssl.SSLError as e: 127 # encapsulate ssl errors 128 raise BackendException(u"SSL failed: %s" % util.uexc(e), 129 log.ErrorCode.backend_error) 130 131 132class WebDAVBackend(duplicity.backend.Backend): 133 u"""Backend for accessing a WebDAV repository. 134 135 webdav backend contributed in 2006 by Jesper Zedlitz <jesper@zedlitz.de> 136 """ 137 138 u""" 139 Request just the names. 140 """ 141 listbody = u'<?xml version="1.0"?><D:propfind xmlns:D="DAV:"><D:prop><D:resourcetype/></D:prop></D:propfind>' 142 143 u"""Connect to remote store using WebDAV Protocol""" 144 def __init__(self, parsed_url): 145 duplicity.backend.Backend.__init__(self, parsed_url) 146 self.headers = {u'Connection': u'keep-alive'} 147 self.parsed_url = parsed_url 148 self.digest_challenge = None 149 self.digest_auth_handler = None 150 151 self.username = parsed_url.username 152 self.password = self.get_password() 153 self.directory = self.sanitize_path(parsed_url.path) 154 155 log.Info(_(u"Using WebDAV protocol %s") % (config.webdav_proto,)) 156 log.Info(_(u"Using WebDAV host %s port %s") % (parsed_url.hostname, 157 parsed_url.port)) 158 log.Info(_(u"Using WebDAV directory %s") % (self.directory,)) 159 160 self.conn = None 161 162 def sanitize_path(self, path): 163 if path: 164 foldpath = re.compile(u'/+') 165 return foldpath.sub(u'/', path + u'/') 166 else: 167 return u'/' 168 169 def getText(self, nodelist): 170 rc = u"" 171 for node in nodelist: 172 if node.nodeType == node.TEXT_NODE: 173 rc = rc + node.data 174 return rc 175 176 def _retry_cleanup(self): 177 self.connect(forced=True) 178 179 def connect(self, forced=False): 180 u""" 181 Connect or re-connect to the server, updates self.conn 182 # reconnect on errors as a precaution, there are errors e.g. 183 # "[Errno 32] Broken pipe" or SSl errors that render the connection unusable 184 """ 185 if not forced and self.conn \ 186 and self.conn.host == self.parsed_url.hostname: 187 return 188 189 log.Info(_(u"WebDAV create connection on '%s'") % (self.parsed_url.hostname)) 190 self._close() 191 # http schemes needed for redirect urls from servers 192 if self.parsed_url.scheme in [u'webdav', u'http']: 193 self.conn = http.client.HTTPConnection(self.parsed_url.hostname, self.parsed_url.port) 194 elif self.parsed_url.scheme in [u'webdavs', u'https']: 195 if config.ssl_no_check_certificate: 196 self.conn = http.client.HTTPSConnection(self.parsed_url.hostname, self.parsed_url.port) 197 else: 198 self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname, self.parsed_url.port) 199 else: 200 raise FatalBackendException(_(u"WebDAV Unknown URI scheme: %s") % (self.parsed_url.scheme)) 201 202 def _close(self): 203 if self.conn: 204 self.conn.close() 205 206 def request(self, method, path, data=None, redirected=0): 207 u""" 208 Wraps the connection.request method to retry once if authentication is 209 required 210 """ 211 self._close() # or we get previous request's data or exception 212 self.connect() 213 214 quoted_path = urllib.parse.quote(path, u"/:~") 215 216 if self.digest_challenge is not None: 217 self.headers[u'Authorization'] = self.get_digest_authorization(path) 218 219 log.Info(_(u"WebDAV %s %s request with headers: %s ") % (method, quoted_path, self.headers)) 220 log.Info(_(u"WebDAV data length: %s ") % len(str(data))) 221 self.conn.request(method, quoted_path, data, self.headers) 222 response = self.conn.getresponse() 223 log.Info(_(u"WebDAV response status %s with reason '%s'.") % (response.status, response.reason)) 224 # resolve redirects and reset url on listing requests (they usually come before everything else) 225 if response.status in [301, 302] and method == u'PROPFIND': 226 redirect_url = response.getheader(u'location', None) 227 response.close() 228 if redirect_url: 229 log.Notice(_(u"WebDAV redirect to: %s ") % urllib.parse.unquote(redirect_url)) 230 if redirected > 10: 231 raise FatalBackendException(_(u"WebDAV redirected 10 times. Giving up.")) 232 self.parsed_url = duplicity.backend.ParsedUrl(redirect_url) 233 self.directory = self.sanitize_path(self.parsed_url.path) 234 return self.request(method, self.directory, data, redirected + 1) 235 else: 236 raise FatalBackendException(_(u"WebDAV missing location header in redirect response.")) 237 elif response.status == 401: 238 response.read() 239 response.close() 240 self.headers[u'Authorization'] = self.get_authorization(response, quoted_path) 241 log.Info(_(u"WebDAV retry request with authentification headers.")) 242 log.Info(_(u"WebDAV %s %s request2 with headers: %s ") % (method, quoted_path, self.headers)) 243 log.Info(_(u"WebDAV data length: %s ") % len(str(data))) 244 self.conn.request(method, quoted_path, data, self.headers) 245 response = self.conn.getresponse() 246 log.Info(_(u"WebDAV response2 status %s with reason '%s'.") % (response.status, response.reason)) 247 248 return response 249 250 def get_authorization(self, response, path): 251 u""" 252 Fetches the auth header based on the requested method (basic or digest) 253 """ 254 try: 255 auth_hdr = response.getheader(u'www-authenticate', u'') 256 token, challenge = auth_hdr.split(u' ', 1) 257 except ValueError: 258 return None 259 if token.split(u',')[0].lower() == u'negotiate': 260 try: 261 return self.get_kerberos_authorization() 262 except ImportError: 263 log.Warn(_(u"python-kerberos needed to use kerberos \ 264 authorization, falling back to basic auth.")) 265 return self.get_basic_authorization() 266 except Exception as e: 267 log.Warn(_(u"Kerberos authorization failed: %s.\ 268 Falling back to basic auth.") % e) 269 return self.get_basic_authorization() 270 elif token.lower() == u'basic': 271 return self.get_basic_authorization() 272 else: 273 self.digest_challenge = self.parse_digest_challenge(challenge) 274 return self.get_digest_authorization(path) 275 276 def parse_digest_challenge(self, challenge_string): 277 return urllib.request.parse_keqv_list(urllib.request.parse_http_list(challenge_string)) 278 279 def get_kerberos_authorization(self): 280 import kerberos # pylint: disable=import-error 281 _, ctx = kerberos.authGSSClientInit(u"HTTP@%s" % self.conn.host) 282 kerberos.authGSSClientStep(ctx, u"") 283 tgt = kerberos.authGSSClientResponse(ctx) 284 return u'Negotiate %s' % tgt 285 286 def get_basic_authorization(self): 287 u""" 288 Returns the basic auth header 289 """ 290 auth_string = u'%s:%s' % (self.username, self.password) 291 return u'Basic %s' % base64.b64encode(auth_string.encode()).strip().decode() 292 293 def get_digest_authorization(self, path): 294 u""" 295 Returns the digest auth header 296 """ 297 u = self.parsed_url 298 if self.digest_auth_handler is None: 299 pw_manager = urllib.request.HTTPPasswordMgrWithDefaultRealm() 300 pw_manager.add_password(None, self.conn.host, self.username, self.password) 301 self.digest_auth_handler = urllib.request.HTTPDigestAuthHandler(pw_manager) 302 303 # building a dummy request that gets never sent, 304 # needed for call to auth_handler.get_authorization 305 scheme = u.scheme == u'webdavs' and u'https' or u'http' 306 hostname = u.port and u"%s:%s" % (u.hostname, u.port) or u.hostname 307 dummy_url = u"%s://%s%s" % (scheme, hostname, path) 308 dummy_req = CustomMethodRequest(self.conn._method, dummy_url) 309 auth_string = self.digest_auth_handler.get_authorization(dummy_req, 310 self.digest_challenge) 311 return u'Digest %s' % auth_string 312 313 def _list(self): 314 response = None 315 try: 316 self.headers[u'Depth'] = u"1" 317 response = self.request(u"PROPFIND", self.directory, self.listbody) 318 del self.headers[u'Depth'] 319 # if the target collection does not exist, create it. 320 if response.status == 404: 321 response.close() # otherwise next request fails with ResponseNotReady 322 self.makedir() 323 # just created an empty folder, so return empty 324 return [] 325 elif response.status in [200, 207]: 326 document = response.read() 327 response.close() 328 else: 329 status = response.status 330 reason = response.reason 331 response.close() 332 raise BackendException(u"Bad status code %s reason %s." % (status, reason)) 333 334 log.Debug(u"%s" % (document,)) 335 dom = xml.dom.minidom.parseString(document) 336 result = [] 337 for href in dom.getElementsByTagName(u'd:href') + dom.getElementsByTagName(u'D:href'): 338 filename = self.taste_href(href) 339 if filename: 340 result.append(filename) 341 return result 342 except Exception as e: 343 raise e 344 finally: 345 if response: 346 response.close() 347 348 def makedir(self): 349 u"""Make (nested) directories on the server.""" 350 dirs = self.directory.split(u"/") 351 # url causes directory to start with /, but it might be given 352 # with or without trailing / (which is required) 353 if dirs[-1] == u'': 354 dirs = dirs[0:-1] 355 for i in range(1, len(dirs)): 356 d = u"/".join(dirs[0:i + 1]) + u"/" 357 358 self.headers[u'Depth'] = u"1" 359 response = self.request(u"PROPFIND", d) 360 del self.headers[u'Depth'] 361 362 log.Info(u"Checking existence dir %s: %d" % (d, response.status)) 363 364 if response.status == 404: 365 log.Info(_(u"Creating missing directory %s") % d) 366 367 res = self.request(u"MKCOL", d) 368 if res.status != 201: 369 raise BackendException(_(u"WebDAV MKCOL %s failed: %s %s") % 370 (d, res.status, res.reason)) 371 372 def taste_href(self, href): 373 u""" 374 Internal helper to taste the given href node and, if 375 it is a duplicity file, collect it as a result file. 376 377 @return: A matching filename, or None if the href did not match. 378 """ 379 raw_filename = self.getText(href.childNodes).strip() 380 parsed_url = urllib.parse.urlparse(urllib.parse.unquote(raw_filename)) 381 filename = parsed_url.path 382 log.Debug(_(u"WebDAV path decoding and translation: " 383 u"%s -> %s") % (raw_filename, filename)) 384 385 # at least one WebDAV server returns files in the form 386 # of full URL:s. this may or may not be 387 # according to the standard, but regardless we 388 # feel we want to bail out if the hostname 389 # does not match until someone has looked into 390 # what the WebDAV protocol mandages. 391 if parsed_url.hostname is not None \ 392 and not (parsed_url.hostname == self.parsed_url.hostname): 393 m = u"Received filename was in the form of a "\ 394 u"full url, but the hostname (%s) did "\ 395 u"not match that of the webdav backend "\ 396 u"url (%s) - aborting as a conservative "\ 397 u"safety measure. If this happens to you, "\ 398 u"please report the problem"\ 399 u"" % (parsed_url.hostname, 400 self.parsed_url.hostname) 401 raise BackendException(m) 402 403 if filename.startswith(self.directory): 404 filename = filename.replace(self.directory, u'', 1) 405 return filename 406 else: 407 return None 408 409 def _get(self, remote_filename, local_path): 410 url = self.directory + util.fsdecode(remote_filename) 411 response = None 412 try: 413 target_file = local_path.open(u"wb") 414 response = self.request(u"GET", url) 415 if response.status == 200: 416 # data=response.read() 417 shutil.copyfileobj(response, target_file) 418 # import hashlib 419 # log.Info("WebDAV GOT %s bytes with md5=%s" % 420 # (len(data),hashlib.md5(data).hexdigest()) ) 421 assert not target_file.close() 422 response.close() 423 else: 424 status = response.status 425 reason = response.reason 426 response.close() 427 raise BackendException(_(u"WebDAV GET Bad status code %s reason %s.") % 428 (status, reason)) 429 except Exception as e: 430 raise e 431 finally: 432 if response: 433 response.close() 434 435 def _put(self, source_path, remote_filename): 436 url = self.directory + util.fsdecode(remote_filename) 437 response = None 438 try: 439 source_file = source_path.open(u"rb") 440 response = self.request(u"PUT", url, source_file.read()) 441 # 200 is returned if a file is overwritten during restarting 442 if response.status in [200, 201, 204]: 443 response.read() 444 response.close() 445 else: 446 status = response.status 447 reason = response.reason 448 response.close() 449 raise BackendException(_(u"WebDAV PUT Bad status code %s reason %s.") % 450 (status, reason)) 451 except Exception as e: 452 raise e 453 finally: 454 if response: 455 response.close() 456 457 def _delete(self, filename): 458 url = self.directory + util.fsdecode(filename) 459 response = None 460 try: 461 response = self.request(u"DELETE", url) 462 if response.status in [200, 204]: 463 response.read() 464 response.close() 465 else: 466 status = response.status 467 reason = response.reason 468 response.close() 469 raise BackendException(_(u"WebDAV DEL Bad status code %s reason %s.") % 470 (status, reason)) 471 except Exception as e: 472 raise e 473 finally: 474 if response: 475 response.close() 476 477 478duplicity.backend.register_backend(u"http", WebDAVBackend) 479duplicity.backend.register_backend(u"https", WebDAVBackend) 480duplicity.backend.register_backend(u"webdav", WebDAVBackend) 481duplicity.backend.register_backend(u"webdavs", WebDAVBackend) 482duplicity.backend.uses_netloc.extend([u'http', u'https', u'webdav', u'webdavs']) 483