1# Copyright (C) 2005-2011, 2016, 2017 Canonical Ltd 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation; either version 2 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 16 17"""Implementation of Transport over SFTP, using paramiko.""" 18 19# TODO: Remove the transport-based lock_read and lock_write methods. They'll 20# then raise TransportNotPossible, which will break remote access to any 21# formats which rely on OS-level locks. That should be fine as those formats 22# are pretty old, but these combinations may have to be removed from the test 23# suite. Those formats all date back to 0.7; so we should be able to remove 24# these methods when we officially drop support for those formats. 25 26import bisect 27import errno 28import itertools 29import os 30import random 31import stat 32import sys 33import time 34import warnings 35 36from .. import ( 37 config, 38 debug, 39 errors, 40 urlutils, 41 ) 42from ..errors import (FileExists, 43 NoSuchFile, 44 TransportError, 45 LockError, 46 PathError, 47 ParamikoNotPresent, 48 ) 49from ..osutils import fancy_rename 50from ..trace import mutter, warning 51from ..transport import ( 52 FileFileStream, 53 _file_streams, 54 ssh, 55 ConnectedTransport, 56 ) 57 58# Disable one particular warning that comes from paramiko in Python2.5; if 59# this is emitted at the wrong time it tends to cause spurious test failures 60# or at least noise in the test case:: 61# 62# [1770/7639 in 86s, 1 known failures, 50 skipped, 2 missing features] 63# test_permissions.TestSftpPermissions.test_new_files 64# /var/lib/python-support/python2.5/paramiko/message.py:226: DeprecationWarning: integer argument expected, got float 65# self.packet.write(struct.pack('>I', n)) 66warnings.filterwarnings('ignore', 67 'integer argument expected, got float', 68 category=DeprecationWarning, 69 module='paramiko.message') 70 71try: 72 import paramiko 73except ImportError as e: 74 raise ParamikoNotPresent(e) 75else: 76 from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE, 77 SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC, 78 CMD_HANDLE, CMD_OPEN) 79 from paramiko.sftp_attr import SFTPAttributes 80 from paramiko.sftp_file import SFTPFile 81 82 83# GZ 2017-05-25: Some dark hackery to monkeypatch out issues with paramiko's 84# Python 3 compatibility code. Replace broken b() and asbytes() code. 85try: 86 from paramiko.py3compat import b as _bad 87 from paramiko.common import asbytes as _bad_asbytes 88except ImportError: 89 pass 90else: 91 def _b_for_broken_paramiko(s, encoding='utf8'): 92 """Hacked b() that does not raise TypeError.""" 93 # https://github.com/paramiko/paramiko/issues/967 94 if not isinstance(s, bytes): 95 encode = getattr(s, 'encode', None) 96 if encode is not None: 97 return encode(encoding) 98 # Would like to pass buffer objects along, but have to realise. 99 tobytes = getattr(s, 'tobytes', None) 100 if tobytes is not None: 101 return tobytes() 102 return s 103 104 def _asbytes_for_broken_paramiko(s): 105 """Hacked asbytes() that does not raise Exception.""" 106 # https://github.com/paramiko/paramiko/issues/968 107 if not isinstance(s, bytes): 108 encode = getattr(s, 'encode', None) 109 if encode is not None: 110 return encode('utf8') 111 asbytes = getattr(s, 'asbytes', None) 112 if asbytes is not None: 113 return asbytes() 114 return s 115 116 _bad.__code__ = _b_for_broken_paramiko.__code__ 117 _bad_asbytes.__code__ = _asbytes_for_broken_paramiko.__code__ 118 119 120class SFTPLock(object): 121 """This fakes a lock in a remote location. 122 123 A present lock is indicated just by the existence of a file. This 124 doesn't work well on all transports and they are only used in 125 deprecated storage formats. 126 """ 127 128 __slots__ = ['path', 'lock_path', 'lock_file', 'transport'] 129 130 def __init__(self, path, transport): 131 self.lock_file = None 132 self.path = path 133 self.lock_path = path + '.write-lock' 134 self.transport = transport 135 try: 136 # RBC 20060103 FIXME should we be using private methods here ? 137 abspath = transport._remote_path(self.lock_path) 138 self.lock_file = transport._sftp_open_exclusive(abspath) 139 except FileExists: 140 raise LockError('File %r already locked' % (self.path,)) 141 142 def unlock(self): 143 if not self.lock_file: 144 return 145 self.lock_file.close() 146 self.lock_file = None 147 try: 148 self.transport.delete(self.lock_path) 149 except (NoSuchFile,): 150 # What specific errors should we catch here? 151 pass 152 153 154class _SFTPReadvHelper(object): 155 """A class to help with managing the state of a readv request.""" 156 157 # See _get_requests for an explanation. 158 _max_request_size = 32768 159 160 def __init__(self, original_offsets, relpath, _report_activity): 161 """Create a new readv helper. 162 163 :param original_offsets: The original requests given by the caller of 164 readv() 165 :param relpath: The name of the file (if known) 166 :param _report_activity: A Transport._report_activity bound method, 167 to be called as data arrives. 168 """ 169 self.original_offsets = list(original_offsets) 170 self.relpath = relpath 171 self._report_activity = _report_activity 172 173 def _get_requests(self): 174 """Break up the offsets into individual requests over sftp. 175 176 The SFTP spec only requires implementers to support 32kB requests. We 177 could try something larger (openssh supports 64kB), but then we have to 178 handle requests that fail. 179 So instead, we just break up our maximum chunks into 32kB chunks, and 180 asyncronously requests them. 181 Newer versions of paramiko would do the chunking for us, but we want to 182 start processing results right away, so we do it ourselves. 183 """ 184 # TODO: Because we issue async requests, we don't 'fudge' any extra 185 # data. I'm not 100% sure that is the best choice. 186 187 # The first thing we do, is to collapse the individual requests as much 188 # as possible, so we don't issues requests <32kB 189 sorted_offsets = sorted(self.original_offsets) 190 coalesced = list(ConnectedTransport._coalesce_offsets(sorted_offsets, 191 limit=0, fudge_factor=0)) 192 requests = [] 193 for c_offset in coalesced: 194 start = c_offset.start 195 size = c_offset.length 196 197 # Break this up into 32kB requests 198 while size > 0: 199 next_size = min(size, self._max_request_size) 200 requests.append((start, next_size)) 201 size -= next_size 202 start += next_size 203 if 'sftp' in debug.debug_flags: 204 mutter('SFTP.readv(%s) %s offsets => %s coalesced => %s requests', 205 self.relpath, len(sorted_offsets), len(coalesced), 206 len(requests)) 207 return requests 208 209 def request_and_yield_offsets(self, fp): 210 """Request the data from the remote machine, yielding the results. 211 212 :param fp: A Paramiko SFTPFile object that supports readv. 213 :return: Yield the data requested by the original readv caller, one by 214 one. 215 """ 216 requests = self._get_requests() 217 offset_iter = iter(self.original_offsets) 218 cur_offset, cur_size = next(offset_iter) 219 # paramiko .readv() yields strings that are in the order of the requests 220 # So we track the current request to know where the next data is 221 # being returned from. 222 input_start = None 223 last_end = None 224 buffered_data = [] 225 buffered_len = 0 226 227 # This is used to buffer chunks which we couldn't process yet 228 # It is (start, end, data) tuples. 229 data_chunks = [] 230 # Create an 'unlimited' data stream, so we stop based on requests, 231 # rather than just because the data stream ended. This lets us detect 232 # short readv. 233 data_stream = itertools.chain(fp.readv(requests), 234 itertools.repeat(None)) 235 for (start, length), data in zip(requests, data_stream): 236 if data is None: 237 if cur_coalesced is not None: 238 raise errors.ShortReadvError(self.relpath, 239 start, length, len(data)) 240 if len(data) != length: 241 raise errors.ShortReadvError(self.relpath, 242 start, length, len(data)) 243 self._report_activity(length, 'read') 244 if last_end is None: 245 # This is the first request, just buffer it 246 buffered_data = [data] 247 buffered_len = length 248 input_start = start 249 elif start == last_end: 250 # The data we are reading fits neatly on the previous 251 # buffer, so this is all part of a larger coalesced range. 252 buffered_data.append(data) 253 buffered_len += length 254 else: 255 # We have an 'interrupt' in the data stream. So we know we are 256 # at a request boundary. 257 if buffered_len > 0: 258 # We haven't consumed the buffer so far, so put it into 259 # data_chunks, and continue. 260 buffered = b''.join(buffered_data) 261 data_chunks.append((input_start, buffered)) 262 input_start = start 263 buffered_data = [data] 264 buffered_len = length 265 last_end = start + length 266 if input_start == cur_offset and cur_size <= buffered_len: 267 # Simplify the next steps a bit by transforming buffered_data 268 # into a single string. We also have the nice property that 269 # when there is only one string ''.join([x]) == x, so there is 270 # no data copying. 271 buffered = b''.join(buffered_data) 272 # Clean out buffered data so that we keep memory 273 # consumption low 274 del buffered_data[:] 275 buffered_offset = 0 276 # TODO: We *could* also consider the case where cur_offset is in 277 # in the buffered range, even though it doesn't *start* 278 # the buffered range. But for packs we pretty much always 279 # read in order, so you won't get any extra data in the 280 # middle. 281 while (input_start == cur_offset 282 and (buffered_offset + cur_size) <= buffered_len): 283 # We've buffered enough data to process this request, spit it 284 # out 285 cur_data = buffered[buffered_offset:buffered_offset + cur_size] 286 # move the direct pointer into our buffered data 287 buffered_offset += cur_size 288 # Move the start-of-buffer pointer 289 input_start += cur_size 290 # Yield the requested data 291 yield cur_offset, cur_data 292 try: 293 cur_offset, cur_size = next(offset_iter) 294 except StopIteration: 295 return 296 # at this point, we've consumed as much of buffered as we can, 297 # so break off the portion that we consumed 298 if buffered_offset == len(buffered_data): 299 # No tail to leave behind 300 buffered_data = [] 301 buffered_len = 0 302 else: 303 buffered = buffered[buffered_offset:] 304 buffered_data = [buffered] 305 buffered_len = len(buffered) 306 # now that the data stream is done, close the handle 307 fp.close() 308 if buffered_len: 309 buffered = b''.join(buffered_data) 310 del buffered_data[:] 311 data_chunks.append((input_start, buffered)) 312 if data_chunks: 313 if 'sftp' in debug.debug_flags: 314 mutter('SFTP readv left with %d out-of-order bytes', 315 sum(len(x[1]) for x in data_chunks)) 316 # We've processed all the readv data, at this point, anything we 317 # couldn't process is in data_chunks. This doesn't happen often, so 318 # this code path isn't optimized 319 # We use an interesting process for data_chunks 320 # Specifically if we have "bisect_left([(start, len, entries)], 321 # (qstart,)]) 322 # If start == qstart, then we get the specific node. Otherwise we 323 # get the previous node 324 while True: 325 idx = bisect.bisect_left(data_chunks, (cur_offset,)) 326 if idx < len(data_chunks) and data_chunks[idx][0] == cur_offset: 327 # The data starts here 328 data = data_chunks[idx][1][:cur_size] 329 elif idx > 0: 330 # The data is in a portion of a previous page 331 idx -= 1 332 sub_offset = cur_offset - data_chunks[idx][0] 333 data = data_chunks[idx][1] 334 data = data[sub_offset:sub_offset + cur_size] 335 else: 336 # We are missing the page where the data should be found, 337 # something is wrong 338 data = '' 339 if len(data) != cur_size: 340 raise AssertionError('We must have miscalulated.' 341 ' We expected %d bytes, but only found %d' 342 % (cur_size, len(data))) 343 yield cur_offset, data 344 try: 345 cur_offset, cur_size = next(offset_iter) 346 except StopIteration: 347 return 348 349 350class SFTPTransport(ConnectedTransport): 351 """Transport implementation for SFTP access.""" 352 353 # TODO: jam 20060717 Conceivably these could be configurable, either 354 # by auto-tuning at run-time, or by a configuration (per host??) 355 # but the performance curve is pretty flat, so just going with 356 # reasonable defaults. 357 _max_readv_combine = 200 358 # Having to round trip to the server means waiting for a response, 359 # so it is better to download extra bytes. 360 # 8KiB had good performance for both local and remote network operations 361 _bytes_to_read_before_seek = 8192 362 363 # The sftp spec says that implementations SHOULD allow reads 364 # to be at least 32K. paramiko.readv() does an async request 365 # for the chunks. So we need to keep it within a single request 366 # size for paramiko <= 1.6.1. paramiko 1.6.2 will probably chop 367 # up the request itself, rather than us having to worry about it 368 _max_request_size = 32768 369 370 def _remote_path(self, relpath): 371 """Return the path to be passed along the sftp protocol for relpath. 372 373 :param relpath: is a urlencoded string. 374 """ 375 remote_path = self._parsed_url.clone(relpath).path 376 # the initial slash should be removed from the path, and treated as a 377 # homedir relative path (the path begins with a double slash if it is 378 # absolute). see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt 379 # RBC 20060118 we are not using this as its too user hostile. instead 380 # we are following lftp and using /~/foo to mean '~/foo' 381 # vila--20070602 and leave absolute paths begin with a single slash. 382 if remote_path.startswith('/~/'): 383 remote_path = remote_path[3:] 384 elif remote_path == '/~': 385 remote_path = '' 386 return remote_path 387 388 def _create_connection(self, credentials=None): 389 """Create a new connection with the provided credentials. 390 391 :param credentials: The credentials needed to establish the connection. 392 393 :return: The created connection and its associated credentials. 394 395 The credentials are only the password as it may have been entered 396 interactively by the user and may be different from the one provided 397 in base url at transport creation time. 398 """ 399 if credentials is None: 400 password = self._parsed_url.password 401 else: 402 password = credentials 403 404 vendor = ssh._get_ssh_vendor() 405 user = self._parsed_url.user 406 if user is None: 407 auth = config.AuthenticationConfig() 408 user = auth.get_user('ssh', self._parsed_url.host, 409 self._parsed_url.port) 410 connection = vendor.connect_sftp(self._parsed_url.user, password, 411 self._parsed_url.host, self._parsed_url.port) 412 return connection, (user, password) 413 414 def disconnect(self): 415 connection = self._get_connection() 416 if connection is not None: 417 connection.close() 418 419 def _get_sftp(self): 420 """Ensures that a connection is established""" 421 connection = self._get_connection() 422 if connection is None: 423 # First connection ever 424 connection, credentials = self._create_connection() 425 self._set_connection(connection, credentials) 426 return connection 427 428 def has(self, relpath): 429 """ 430 Does the target location exist? 431 """ 432 try: 433 self._get_sftp().stat(self._remote_path(relpath)) 434 # stat result is about 20 bytes, let's say 435 self._report_activity(20, 'read') 436 return True 437 except IOError: 438 return False 439 440 def get(self, relpath): 441 """Get the file at the given relative path. 442 443 :param relpath: The relative path to the file 444 """ 445 try: 446 path = self._remote_path(relpath) 447 f = self._get_sftp().file(path, mode='rb') 448 size = f.stat().st_size 449 if getattr(f, 'prefetch', None) is not None: 450 f.prefetch(size) 451 return f 452 except (IOError, paramiko.SSHException) as e: 453 self._translate_io_exception(e, path, ': error retrieving', 454 failure_exc=errors.ReadError) 455 456 def get_bytes(self, relpath): 457 # reimplement this here so that we can report how many bytes came back 458 with self.get(relpath) as f: 459 bytes = f.read() 460 self._report_activity(len(bytes), 'read') 461 return bytes 462 463 def _readv(self, relpath, offsets): 464 """See Transport.readv()""" 465 # We overload the default readv() because we want to use a file 466 # that does not have prefetch enabled. 467 # Also, if we have a new paramiko, it implements an async readv() 468 if not offsets: 469 return 470 471 try: 472 path = self._remote_path(relpath) 473 fp = self._get_sftp().file(path, mode='rb') 474 readv = getattr(fp, 'readv', None) 475 if readv: 476 return self._sftp_readv(fp, offsets, relpath) 477 if 'sftp' in debug.debug_flags: 478 mutter('seek and read %s offsets', len(offsets)) 479 return self._seek_and_read(fp, offsets, relpath) 480 except (IOError, paramiko.SSHException) as e: 481 self._translate_io_exception(e, path, ': error retrieving') 482 483 def recommended_page_size(self): 484 """See Transport.recommended_page_size(). 485 486 For SFTP we suggest a large page size to reduce the overhead 487 introduced by latency. 488 """ 489 return 64 * 1024 490 491 def _sftp_readv(self, fp, offsets, relpath): 492 """Use the readv() member of fp to do async readv. 493 494 Then read them using paramiko.readv(). paramiko.readv() 495 does not support ranges > 64K, so it caps the request size, and 496 just reads until it gets all the stuff it wants. 497 """ 498 helper = _SFTPReadvHelper(offsets, relpath, self._report_activity) 499 return helper.request_and_yield_offsets(fp) 500 501 def put_file(self, relpath, f, mode=None): 502 """ 503 Copy the file-like object into the location. 504 505 :param relpath: Location to put the contents, relative to base. 506 :param f: File-like object. 507 :param mode: The final mode for the file 508 """ 509 final_path = self._remote_path(relpath) 510 return self._put(final_path, f, mode=mode) 511 512 def _put(self, abspath, f, mode=None): 513 """Helper function so both put() and copy_abspaths can reuse the code""" 514 tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(), 515 os.getpid(), random.randint(0, 0x7FFFFFFF)) 516 fout = self._sftp_open_exclusive(tmp_abspath, mode=mode) 517 closed = False 518 try: 519 try: 520 fout.set_pipelined(True) 521 length = self._pump(f, fout) 522 except (IOError, paramiko.SSHException) as e: 523 self._translate_io_exception(e, tmp_abspath) 524 # XXX: This doesn't truly help like we would like it to. 525 # The problem is that openssh strips sticky bits. So while we 526 # can properly set group write permission, we lose the group 527 # sticky bit. So it is probably best to stop chmodding, and 528 # just tell users that they need to set the umask correctly. 529 # The attr.st_mode = mode, in _sftp_open_exclusive 530 # will handle when the user wants the final mode to be more 531 # restrictive. And then we avoid a round trip. Unless 532 # paramiko decides to expose an async chmod() 533 534 # This is designed to chmod() right before we close. 535 # Because we set_pipelined() earlier, theoretically we might 536 # avoid the round trip for fout.close() 537 if mode is not None: 538 self._get_sftp().chmod(tmp_abspath, mode) 539 fout.close() 540 closed = True 541 self._rename_and_overwrite(tmp_abspath, abspath) 542 return length 543 except Exception as e: 544 # If we fail, try to clean up the temporary file 545 # before we throw the exception 546 # but don't let another exception mess things up 547 # Write out the traceback, because otherwise 548 # the catch and throw destroys it 549 import traceback 550 mutter(traceback.format_exc()) 551 try: 552 if not closed: 553 fout.close() 554 self._get_sftp().remove(tmp_abspath) 555 except: 556 # raise the saved except 557 raise e 558 # raise the original with its traceback if we can. 559 raise 560 561 def _put_non_atomic_helper(self, relpath, writer, mode=None, 562 create_parent_dir=False, 563 dir_mode=None): 564 abspath = self._remote_path(relpath) 565 566 # TODO: jam 20060816 paramiko doesn't publicly expose a way to 567 # set the file mode at create time. If it does, use it. 568 # But for now, we just chmod later anyway. 569 570 def _open_and_write_file(): 571 """Try to open the target file, raise error on failure""" 572 fout = None 573 try: 574 try: 575 fout = self._get_sftp().file(abspath, mode='wb') 576 fout.set_pipelined(True) 577 writer(fout) 578 except (paramiko.SSHException, IOError) as e: 579 self._translate_io_exception(e, abspath, 580 ': unable to open') 581 582 # This is designed to chmod() right before we close. 583 # Because we set_pipelined() earlier, theoretically we might 584 # avoid the round trip for fout.close() 585 if mode is not None: 586 self._get_sftp().chmod(abspath, mode) 587 finally: 588 if fout is not None: 589 fout.close() 590 591 if not create_parent_dir: 592 _open_and_write_file() 593 return 594 595 # Try error handling to create the parent directory if we need to 596 try: 597 _open_and_write_file() 598 except NoSuchFile: 599 # Try to create the parent directory, and then go back to 600 # writing the file 601 parent_dir = os.path.dirname(abspath) 602 self._mkdir(parent_dir, dir_mode) 603 _open_and_write_file() 604 605 def put_file_non_atomic(self, relpath, f, mode=None, 606 create_parent_dir=False, 607 dir_mode=None): 608 """Copy the file-like object into the target location. 609 610 This function is not strictly safe to use. It is only meant to 611 be used when you already know that the target does not exist. 612 It is not safe, because it will open and truncate the remote 613 file. So there may be a time when the file has invalid contents. 614 615 :param relpath: The remote location to put the contents. 616 :param f: File-like object. 617 :param mode: Possible access permissions for new file. 618 None means do not set remote permissions. 619 :param create_parent_dir: If we cannot create the target file because 620 the parent directory does not exist, go ahead and 621 create it, and then try again. 622 """ 623 def writer(fout): 624 self._pump(f, fout) 625 self._put_non_atomic_helper(relpath, writer, mode=mode, 626 create_parent_dir=create_parent_dir, 627 dir_mode=dir_mode) 628 629 def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None, 630 create_parent_dir=False, 631 dir_mode=None): 632 if not isinstance(raw_bytes, bytes): 633 raise TypeError( 634 'raw_bytes must be a plain string, not %s' % type(raw_bytes)) 635 636 def writer(fout): 637 fout.write(raw_bytes) 638 self._put_non_atomic_helper(relpath, writer, mode=mode, 639 create_parent_dir=create_parent_dir, 640 dir_mode=dir_mode) 641 642 def iter_files_recursive(self): 643 """Walk the relative paths of all files in this transport.""" 644 # progress is handled by list_dir 645 queue = list(self.list_dir('.')) 646 while queue: 647 relpath = queue.pop(0) 648 st = self.stat(relpath) 649 if stat.S_ISDIR(st.st_mode): 650 for i, basename in enumerate(self.list_dir(relpath)): 651 queue.insert(i, relpath + '/' + basename) 652 else: 653 yield relpath 654 655 def _mkdir(self, abspath, mode=None): 656 if mode is None: 657 local_mode = 0o777 658 else: 659 local_mode = mode 660 try: 661 self._report_activity(len(abspath), 'write') 662 self._get_sftp().mkdir(abspath, local_mode) 663 self._report_activity(1, 'read') 664 if mode is not None: 665 # chmod a dir through sftp will erase any sgid bit set 666 # on the server side. So, if the bit mode are already 667 # set, avoid the chmod. If the mode is not fine but 668 # the sgid bit is set, report a warning to the user 669 # with the umask fix. 670 stat = self._get_sftp().lstat(abspath) 671 mode = mode & 0o777 # can't set special bits anyway 672 if mode != stat.st_mode & 0o777: 673 if stat.st_mode & 0o6000: 674 warning('About to chmod %s over sftp, which will result' 675 ' in its suid or sgid bits being cleared. If' 676 ' you want to preserve those bits, change your ' 677 ' environment on the server to use umask 0%03o.' 678 % (abspath, 0o777 - mode)) 679 self._get_sftp().chmod(abspath, mode=mode) 680 except (paramiko.SSHException, IOError) as e: 681 self._translate_io_exception(e, abspath, ': unable to mkdir', 682 failure_exc=FileExists) 683 684 def mkdir(self, relpath, mode=None): 685 """Create a directory at the given path.""" 686 self._mkdir(self._remote_path(relpath), mode=mode) 687 688 def open_write_stream(self, relpath, mode=None): 689 """See Transport.open_write_stream.""" 690 # initialise the file to zero-length 691 # this is three round trips, but we don't use this 692 # api more than once per write_group at the moment so 693 # it is a tolerable overhead. Better would be to truncate 694 # the file after opening. RBC 20070805 695 self.put_bytes_non_atomic(relpath, b"", mode) 696 abspath = self._remote_path(relpath) 697 # TODO: jam 20060816 paramiko doesn't publicly expose a way to 698 # set the file mode at create time. If it does, use it. 699 # But for now, we just chmod later anyway. 700 handle = None 701 try: 702 handle = self._get_sftp().file(abspath, mode='wb') 703 handle.set_pipelined(True) 704 except (paramiko.SSHException, IOError) as e: 705 self._translate_io_exception(e, abspath, 706 ': unable to open') 707 _file_streams[self.abspath(relpath)] = handle 708 return FileFileStream(self, relpath, handle) 709 710 def _translate_io_exception(self, e, path, more_info='', 711 failure_exc=PathError): 712 """Translate a paramiko or IOError into a friendlier exception. 713 714 :param e: The original exception 715 :param path: The path in question when the error is raised 716 :param more_info: Extra information that can be included, 717 such as what was going on 718 :param failure_exc: Paramiko has the super fun ability to raise completely 719 opaque errors that just set "e.args = ('Failure',)" with 720 no more information. 721 If this parameter is set, it defines the exception 722 to raise in these cases. 723 """ 724 # paramiko seems to generate detailless errors. 725 self._translate_error(e, path, raise_generic=False) 726 if getattr(e, 'args', None) is not None: 727 if (e.args == ('No such file or directory',) or 728 e.args == ('No such file',)): 729 raise NoSuchFile(path, str(e) + more_info) 730 if (e.args == ('mkdir failed',) or 731 e.args[0].startswith('syserr: File exists')): 732 raise FileExists(path, str(e) + more_info) 733 # strange but true, for the paramiko server. 734 if (e.args == ('Failure',)): 735 raise failure_exc(path, str(e) + more_info) 736 # Can be something like args = ('Directory not empty: 737 # '/srv/bazaar.launchpad.net/blah...: ' 738 # [Errno 39] Directory not empty',) 739 if (e.args[0].startswith('Directory not empty: ') 740 or getattr(e, 'errno', None) == errno.ENOTEMPTY): 741 raise errors.DirectoryNotEmpty(path, str(e)) 742 if e.args == ('Operation unsupported',): 743 raise errors.TransportNotPossible() 744 mutter('Raising exception with args %s', e.args) 745 if getattr(e, 'errno', None) is not None: 746 mutter('Raising exception with errno %s', e.errno) 747 raise e 748 749 def append_file(self, relpath, f, mode=None): 750 """ 751 Append the text in the file-like object into the final 752 location. 753 """ 754 try: 755 path = self._remote_path(relpath) 756 fout = self._get_sftp().file(path, 'ab') 757 if mode is not None: 758 self._get_sftp().chmod(path, mode) 759 result = fout.tell() 760 self._pump(f, fout) 761 return result 762 except (IOError, paramiko.SSHException) as e: 763 self._translate_io_exception(e, relpath, ': unable to append') 764 765 def rename(self, rel_from, rel_to): 766 """Rename without special overwriting""" 767 try: 768 self._get_sftp().rename(self._remote_path(rel_from), 769 self._remote_path(rel_to)) 770 except (IOError, paramiko.SSHException) as e: 771 self._translate_io_exception(e, rel_from, 772 ': unable to rename to %r' % (rel_to)) 773 774 def _rename_and_overwrite(self, abs_from, abs_to): 775 """Do a fancy rename on the remote server. 776 777 Using the implementation provided by osutils. 778 """ 779 try: 780 sftp = self._get_sftp() 781 fancy_rename(abs_from, abs_to, 782 rename_func=sftp.rename, 783 unlink_func=sftp.remove) 784 except (IOError, paramiko.SSHException) as e: 785 self._translate_io_exception(e, abs_from, 786 ': unable to rename to %r' % (abs_to)) 787 788 def move(self, rel_from, rel_to): 789 """Move the item at rel_from to the location at rel_to""" 790 path_from = self._remote_path(rel_from) 791 path_to = self._remote_path(rel_to) 792 self._rename_and_overwrite(path_from, path_to) 793 794 def delete(self, relpath): 795 """Delete the item at relpath""" 796 path = self._remote_path(relpath) 797 try: 798 self._get_sftp().remove(path) 799 except (IOError, paramiko.SSHException) as e: 800 self._translate_io_exception(e, path, ': unable to delete') 801 802 def external_url(self): 803 """See breezy.transport.Transport.external_url.""" 804 # the external path for SFTP is the base 805 return self.base 806 807 def listable(self): 808 """Return True if this store supports listing.""" 809 return True 810 811 def list_dir(self, relpath): 812 """ 813 Return a list of all files at the given location. 814 """ 815 # does anything actually use this? 816 # -- Unknown 817 # This is at least used by copy_tree for remote upgrades. 818 # -- David Allouche 2006-08-11 819 path = self._remote_path(relpath) 820 try: 821 entries = self._get_sftp().listdir(path) 822 self._report_activity(sum(map(len, entries)), 'read') 823 except (IOError, paramiko.SSHException) as e: 824 self._translate_io_exception(e, path, ': failed to list_dir') 825 return [urlutils.escape(entry) for entry in entries] 826 827 def rmdir(self, relpath): 828 """See Transport.rmdir.""" 829 path = self._remote_path(relpath) 830 try: 831 return self._get_sftp().rmdir(path) 832 except (IOError, paramiko.SSHException) as e: 833 self._translate_io_exception(e, path, ': failed to rmdir') 834 835 def stat(self, relpath): 836 """Return the stat information for a file.""" 837 path = self._remote_path(relpath) 838 try: 839 return self._get_sftp().lstat(path) 840 except (IOError, paramiko.SSHException) as e: 841 self._translate_io_exception(e, path, ': unable to stat') 842 843 def readlink(self, relpath): 844 """See Transport.readlink.""" 845 path = self._remote_path(relpath) 846 try: 847 return self._get_sftp().readlink(self._remote_path(path)) 848 except (IOError, paramiko.SSHException) as e: 849 self._translate_io_exception(e, path, ': unable to readlink') 850 851 def symlink(self, source, link_name): 852 """See Transport.symlink.""" 853 try: 854 conn = self._get_sftp() 855 sftp_retval = conn.symlink(source, self._remote_path(link_name)) 856 except (IOError, paramiko.SSHException) as e: 857 self._translate_io_exception(e, link_name, 858 ': unable to create symlink to %r' % (source)) 859 860 def lock_read(self, relpath): 861 """ 862 Lock the given file for shared (read) access. 863 :return: A lock object, which has an unlock() member function 864 """ 865 # FIXME: there should be something clever i can do here... 866 class BogusLock(object): 867 def __init__(self, path): 868 self.path = path 869 870 def unlock(self): 871 pass 872 873 def __exit__(self, exc_type, exc_val, exc_tb): 874 return False 875 876 def __enter__(self): 877 pass 878 return BogusLock(relpath) 879 880 def lock_write(self, relpath): 881 """ 882 Lock the given file for exclusive (write) access. 883 WARNING: many transports do not support this, so trying avoid using it 884 885 :return: A lock object, which has an unlock() member function 886 """ 887 # This is a little bit bogus, but basically, we create a file 888 # which should not already exist, and if it does, we assume 889 # that there is a lock, and if it doesn't, the we assume 890 # that we have taken the lock. 891 return SFTPLock(relpath, self) 892 893 def _sftp_open_exclusive(self, abspath, mode=None): 894 """Open a remote path exclusively. 895 896 SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if 897 the file already exists. However it does not expose this 898 at the higher level of SFTPClient.open(), so we have to 899 sneak away with it. 900 901 WARNING: This breaks the SFTPClient abstraction, so it 902 could easily break against an updated version of paramiko. 903 904 :param abspath: The remote absolute path where the file should be opened 905 :param mode: The mode permissions bits for the new file 906 """ 907 # TODO: jam 20060816 Paramiko >= 1.6.2 (probably earlier) supports 908 # using the 'x' flag to indicate SFTP_FLAG_EXCL. 909 # However, there is no way to set the permission mode at open 910 # time using the sftp_client.file() functionality. 911 path = self._get_sftp()._adjust_cwd(abspath) 912 # mutter('sftp abspath %s => %s', abspath, path) 913 attr = SFTPAttributes() 914 if mode is not None: 915 attr.st_mode = mode 916 omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE 917 | SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL) 918 try: 919 t, msg = self._get_sftp()._request(CMD_OPEN, path, omode, attr) 920 if t != CMD_HANDLE: 921 raise TransportError('Expected an SFTP handle') 922 handle = msg.get_string() 923 return SFTPFile(self._get_sftp(), handle, 'wb', -1) 924 except (paramiko.SSHException, IOError) as e: 925 self._translate_io_exception(e, abspath, ': unable to open', 926 failure_exc=FileExists) 927 928 def _can_roundtrip_unix_modebits(self): 929 if sys.platform == 'win32': 930 # anyone else? 931 return False 932 else: 933 return True 934 935 936def get_test_permutations(): 937 """Return the permutations to be used in testing.""" 938 from ..tests import stub_sftp 939 return [(SFTPTransport, stub_sftp.SFTPAbsoluteServer), 940 (SFTPTransport, stub_sftp.SFTPHomeDirServer), 941 (SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer), 942 ] 943