1# Copyright (C) 2005-2012, 2016 Canonical Ltd 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation; either version 2 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 16 17"""Transport is an abstraction layer to handle file access. 18 19The abstraction is to allow access from the local filesystem, as well 20as remote (such as http or sftp). 21 22Transports are constructed from a string, being a URL or (as a degenerate 23case) a local filesystem path. This is typically the top directory of 24a bzrdir, repository, or similar object we are interested in working with. 25The Transport returned has methods to read, write and manipulate files within 26it. 27""" 28 29import errno 30from io import BytesIO 31import sys 32 33from stat import S_ISDIR 34 35from ..trace import ( 36 mutter, 37 ) 38from .. import ( 39 errors, 40 hooks, 41 registry, 42 osutils, 43 ui, 44 urlutils, 45 ) 46 47 48# a dictionary of open file streams. Keys are absolute paths, values are 49# transport defined. 50_file_streams = {} 51 52 53def _get_protocol_handlers(): 54 """Return a dictionary of {urlprefix: [factory]}""" 55 return transport_list_registry 56 57 58def _set_protocol_handlers(new_handlers): 59 """Replace the current protocol handlers dictionary. 60 61 WARNING this will remove all build in protocols. Use with care. 62 """ 63 global transport_list_registry 64 transport_list_registry = new_handlers 65 66 67def _clear_protocol_handlers(): 68 global transport_list_registry 69 transport_list_registry = TransportListRegistry() 70 71 72def _get_transport_modules(): 73 """Return a list of the modules providing transports.""" 74 modules = set() 75 for prefix, factory_list in transport_list_registry.items(): 76 for factory in factory_list: 77 modules.add(factory.get_module()) 78 # Add chroot and pathfilter directly, because there is no handler 79 # registered for it. 80 modules.add('breezy.transport.chroot') 81 modules.add('breezy.transport.pathfilter') 82 result = sorted(modules) 83 return result 84 85 86class UnusableRedirect(errors.BzrError): 87 88 _fmt = ("Unable to follow redirect from %(source)s to %(target)s: " 89 "%(reason)s.") 90 91 def __init__(self, source, target, reason): 92 super(UnusableRedirect, self).__init__( 93 source=source, target=target, reason=reason) 94 95 96class TransportListRegistry(registry.Registry): 97 """A registry which simplifies tracking available Transports. 98 99 A registration of a new protocol requires two steps: 100 1) register the prefix with the function register_transport( ) 101 2) register the protocol provider with the function 102 register_transport_provider( ) ( and the "lazy" variant ) 103 104 This is needed because: 105 a) a single provider can support multiple protocols (like the ftp 106 provider which supports both the ftp:// and the aftp:// protocols) 107 b) a single protocol can have multiple providers (like the http:// 108 protocol which was supported by both the urllib and pycurl providers) 109 """ 110 111 def register_transport_provider(self, key, obj): 112 self.get(key).insert(0, registry._ObjectGetter(obj)) 113 114 def register_lazy_transport_provider(self, key, module_name, member_name): 115 self.get(key).insert(0, 116 registry._LazyObjectGetter(module_name, member_name)) 117 118 def register_transport(self, key, help=None): 119 self.register(key, [], help) 120 121 122transport_list_registry = TransportListRegistry() 123 124 125def register_transport_proto(prefix, help=None, info=None, 126 register_netloc=False): 127 transport_list_registry.register_transport(prefix, help) 128 if register_netloc: 129 if not prefix.endswith('://'): 130 raise ValueError(prefix) 131 register_urlparse_netloc_protocol(prefix[:-3]) 132 133 134def register_lazy_transport(prefix, module, classname): 135 if prefix not in transport_list_registry: 136 register_transport_proto(prefix) 137 transport_list_registry.register_lazy_transport_provider( 138 prefix, module, classname) 139 140 141def register_transport(prefix, klass): 142 if prefix not in transport_list_registry: 143 register_transport_proto(prefix) 144 transport_list_registry.register_transport_provider(prefix, klass) 145 146 147def register_urlparse_netloc_protocol(protocol): 148 """Ensure that protocol is setup to be used with urlparse netloc parsing.""" 149 if protocol not in urlutils.urlparse.uses_netloc: 150 urlutils.urlparse.uses_netloc.append(protocol) 151 152 153def _unregister_urlparse_netloc_protocol(protocol): 154 """Remove protocol from urlparse netloc parsing. 155 156 Except for tests, you should never use that function. Using it with 'http', 157 for example, will break all http transports. 158 """ 159 if protocol in urlutils.urlparse.uses_netloc: 160 urlutils.urlparse.uses_netloc.remove(protocol) 161 162 163def unregister_transport(scheme, factory): 164 """Unregister a transport.""" 165 l = transport_list_registry.get(scheme) 166 for i in l: 167 o = i.get_obj() 168 if o == factory: 169 transport_list_registry.get(scheme).remove(i) 170 break 171 if len(l) == 0: 172 transport_list_registry.remove(scheme) 173 174 175class _CoalescedOffset(object): 176 """A data container for keeping track of coalesced offsets.""" 177 178 __slots__ = ['start', 'length', 'ranges'] 179 180 def __init__(self, start, length, ranges): 181 self.start = start 182 self.length = length 183 self.ranges = ranges 184 185 def __cmp__(self, other): 186 return cmp((self.start, self.length, self.ranges), 187 (other.start, other.length, other.ranges)) 188 189 def __eq__(self, other): 190 return ((self.start, self.length, self.ranges) == 191 (other.start, other.length, other.ranges)) 192 193 def __repr__(self): 194 return '%s(%r, %r, %r)' % (self.__class__.__name__, 195 self.start, self.length, self.ranges) 196 197 198class LateReadError(object): 199 """A helper for transports which pretends to be a readable file. 200 201 When read() is called, errors.ReadError is raised. 202 """ 203 204 def __init__(self, path): 205 self._path = path 206 207 def close(self): 208 """a no-op - do nothing.""" 209 210 def __enter__(self): 211 return self 212 213 def __exit__(self, exc_type, exc_val, exc_tb): 214 # If there was an error raised, prefer the original one 215 try: 216 self.close() 217 except: 218 if exc_type is None: 219 raise 220 return False 221 222 def _fail(self): 223 """Raise ReadError.""" 224 raise errors.ReadError(self._path) 225 226 def __iter__(self): 227 self._fail() 228 229 def read(self, count=-1): 230 self._fail() 231 232 def readlines(self): 233 self._fail() 234 235 236class FileStream(object): 237 """Base class for FileStreams.""" 238 239 def __init__(self, transport, relpath): 240 """Create a FileStream for relpath on transport.""" 241 self.transport = transport 242 self.relpath = relpath 243 244 def _close(self): 245 """A hook point for subclasses that need to take action on close.""" 246 247 def __enter__(self): 248 return self 249 250 def __exit__(self, exc_type, exc_value, exc_tb): 251 self.close() 252 return False 253 254 def close(self, want_fdatasync=False): 255 if want_fdatasync: 256 try: 257 self.fdatasync() 258 except errors.TransportNotPossible: 259 pass 260 self._close() 261 del _file_streams[self.transport.abspath(self.relpath)] 262 263 def fdatasync(self): 264 """Force data out to physical disk if possible. 265 266 :raises TransportNotPossible: If this transport has no way to 267 flush to disk. 268 """ 269 raise errors.TransportNotPossible( 270 "%s cannot fdatasync" % (self.transport,)) 271 272 273class FileFileStream(FileStream): 274 """A file stream object returned by open_write_stream. 275 276 This version uses a file like object to perform writes. 277 """ 278 279 def __init__(self, transport, relpath, file_handle): 280 FileStream.__init__(self, transport, relpath) 281 self.file_handle = file_handle 282 283 def _close(self): 284 self.file_handle.close() 285 286 def fdatasync(self): 287 """Force data out to physical disk if possible.""" 288 self.file_handle.flush() 289 try: 290 fileno = self.file_handle.fileno() 291 except AttributeError: 292 raise errors.TransportNotPossible() 293 osutils.fdatasync(fileno) 294 295 def write(self, bytes): 296 osutils.pump_string_file(bytes, self.file_handle) 297 298 299class AppendBasedFileStream(FileStream): 300 """A file stream object returned by open_write_stream. 301 302 This version uses append on a transport to perform writes. 303 """ 304 305 def write(self, bytes): 306 self.transport.append_bytes(self.relpath, bytes) 307 308 309class TransportHooks(hooks.Hooks): 310 """Mapping of hook names to registered callbacks for transport hooks""" 311 312 def __init__(self): 313 super(TransportHooks, self).__init__() 314 self.add_hook("post_connect", 315 "Called after a new connection is established or a reconnect " 316 "occurs. The sole argument passed is either the connected " 317 "transport or smart medium instance.", (2, 5)) 318 319 320class Transport(object): 321 """This class encapsulates methods for retrieving or putting a file 322 from/to a storage location. 323 324 :ivar base: Base URL for the transport; should always end in a slash. 325 """ 326 327 # implementations can override this if it is more efficient 328 # for them to combine larger read chunks together 329 _max_readv_combine = 50 330 # It is better to read this much more data in order, rather 331 # than doing another seek. Even for the local filesystem, 332 # there is a benefit in just reading. 333 # TODO: jam 20060714 Do some real benchmarking to figure out 334 # where the biggest benefit between combining reads and 335 # and seeking is. Consider a runtime auto-tune. 336 _bytes_to_read_before_seek = 0 337 338 hooks = TransportHooks() 339 340 def __init__(self, base): 341 super(Transport, self).__init__() 342 self.base = base 343 (self._raw_base, self._segment_parameters) = ( 344 urlutils.split_segment_parameters(base)) 345 346 def _translate_error(self, e, path, raise_generic=True): 347 """Translate an IOError or OSError into an appropriate bzr error. 348 349 This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS 350 """ 351 if getattr(e, 'errno', None) is not None: 352 if e.errno in (errno.ENOENT, errno.ENOTDIR): 353 raise errors.NoSuchFile(path, extra=e) 354 elif e.errno == errno.EINVAL: 355 mutter("EINVAL returned on path %s: %r" % (path, e)) 356 raise errors.NoSuchFile(path, extra=e) 357 # I would rather use errno.EFOO, but there doesn't seem to be 358 # any matching for 267 359 # This is the error when doing a listdir on a file: 360 # WindowsError: [Errno 267] The directory name is invalid 361 if sys.platform == 'win32' and e.errno in (errno.ESRCH, 267): 362 raise errors.NoSuchFile(path, extra=e) 363 if e.errno == errno.EEXIST: 364 raise errors.FileExists(path, extra=e) 365 if e.errno == errno.EACCES: 366 raise errors.PermissionDenied(path, extra=e) 367 if e.errno == errno.ENOTEMPTY: 368 raise errors.DirectoryNotEmpty(path, extra=e) 369 if e.errno == errno.EBUSY: 370 raise errors.ResourceBusy(path, extra=e) 371 if raise_generic: 372 raise errors.TransportError(orig_error=e) 373 374 def clone(self, offset=None): 375 """Return a new Transport object, cloned from the current location, 376 using a subdirectory or parent directory. This allows connections 377 to be pooled, rather than a new one needed for each subdir. 378 """ 379 raise NotImplementedError(self.clone) 380 381 def create_prefix(self, mode=None): 382 """Create all the directories leading down to self.base.""" 383 cur_transport = self 384 needed = [cur_transport] 385 # Recurse upwards until we can create a directory successfully 386 while True: 387 new_transport = cur_transport.clone('..') 388 if new_transport.base == cur_transport.base: 389 raise errors.CommandError( 390 "Failed to create path prefix for %s." 391 % cur_transport.base) 392 try: 393 new_transport.mkdir('.', mode=mode) 394 except errors.NoSuchFile: 395 needed.append(new_transport) 396 cur_transport = new_transport 397 except errors.FileExists: 398 break 399 else: 400 break 401 # Now we only need to create child directories 402 while needed: 403 cur_transport = needed.pop() 404 cur_transport.ensure_base(mode=mode) 405 406 def ensure_base(self, mode=None): 407 """Ensure that the directory this transport references exists. 408 409 This will create a directory if it doesn't exist. 410 :return: True if the directory was created, False otherwise. 411 """ 412 # The default implementation just uses "Easier to ask for forgiveness 413 # than permission". We attempt to create the directory, and just 414 # suppress FileExists and PermissionDenied (for Windows) exceptions. 415 try: 416 self.mkdir('.', mode=mode) 417 except (errors.FileExists, errors.PermissionDenied): 418 return False 419 else: 420 return True 421 422 def external_url(self): 423 """Return a URL for self that can be given to an external process. 424 425 There is no guarantee that the URL can be accessed from a different 426 machine - e.g. file:/// urls are only usable on the local machine, 427 sftp:/// urls when the server is only bound to localhost are only 428 usable from localhost etc. 429 430 NOTE: This method may remove security wrappers (e.g. on chroot 431 transports) and thus should *only* be used when the result will not 432 be used to obtain a new transport within breezy. Ideally chroot 433 transports would know enough to cause the external url to be the exact 434 one used that caused the chrooting in the first place, but that is not 435 currently the case. 436 437 :return: A URL that can be given to another process. 438 :raises InProcessTransport: If the transport is one that cannot be 439 accessed out of the current process (e.g. a MemoryTransport) 440 then InProcessTransport is raised. 441 """ 442 raise NotImplementedError(self.external_url) 443 444 def get_segment_parameters(self): 445 """Return the segment parameters for the top segment of the URL. 446 """ 447 return self._segment_parameters 448 449 def set_segment_parameter(self, name, value): 450 """Set a segment parameter. 451 452 :param name: Segment parameter name (urlencoded string) 453 :param value: Segment parameter value (urlencoded string) 454 """ 455 if value is None: 456 try: 457 del self._segment_parameters[name] 458 except KeyError: 459 pass 460 else: 461 self._segment_parameters[name] = value 462 self.base = urlutils.join_segment_parameters( 463 self._raw_base, self._segment_parameters) 464 465 def _pump(self, from_file, to_file): 466 """Most children will need to copy from one file-like 467 object or string to another one. 468 This just gives them something easy to call. 469 """ 470 return osutils.pumpfile(from_file, to_file) 471 472 def _get_total(self, multi): 473 """Try to figure out how many entries are in multi, 474 but if not possible, return None. 475 """ 476 try: 477 return len(multi) 478 except TypeError: # We can't tell how many, because relpaths is a generator 479 return None 480 481 def _report_activity(self, bytes, direction): 482 """Notify that this transport has activity. 483 484 Implementations should call this from all methods that actually do IO. 485 Be careful that it's not called twice, if one method is implemented on 486 top of another. 487 488 :param bytes: Number of bytes read or written. 489 :param direction: 'read' or 'write' or None. 490 """ 491 ui.ui_factory.report_transport_activity(self, bytes, direction) 492 493 def _update_pb(self, pb, msg, count, total): 494 """Update the progress bar based on the current count 495 and total available, total may be None if it was 496 not possible to determine. 497 """ 498 if pb is None: 499 return 500 if total is None: 501 pb.update(msg, count, count + 1) 502 else: 503 pb.update(msg, count, total) 504 505 def _iterate_over(self, multi, func, pb, msg, expand=True): 506 """Iterate over all entries in multi, passing them to func, 507 and update the progress bar as you go along. 508 509 :param expand: If True, the entries will be passed to the function 510 by expanding the tuple. If False, it will be passed 511 as a single parameter. 512 """ 513 total = self._get_total(multi) 514 result = [] 515 count = 0 516 for entry in multi: 517 self._update_pb(pb, msg, count, total) 518 if expand: 519 result.append(func(*entry)) 520 else: 521 result.append(func(entry)) 522 count += 1 523 return tuple(result) 524 525 def abspath(self, relpath): 526 """Return the full url to the given relative path. 527 528 :param relpath: a string of a relative path 529 """ 530 531 # XXX: Robert Collins 20051016 - is this really needed in the public 532 # interface ? 533 raise NotImplementedError(self.abspath) 534 535 def recommended_page_size(self): 536 """Return the recommended page size for this transport. 537 538 This is potentially different for every path in a given namespace. 539 For example, local transports might use an operating system call to 540 get the block size for a given path, which can vary due to mount 541 points. 542 543 :return: The page size in bytes. 544 """ 545 return 4 * 1024 546 547 def relpath(self, abspath): 548 """Return the local path portion from a given absolute path. 549 550 This default implementation is not suitable for filesystems with 551 aliasing, such as that given by symlinks, where a path may not 552 start with our base, but still be a relpath once aliasing is 553 resolved. 554 """ 555 # TODO: This might want to use breezy.osutils.relpath 556 # but we have to watch out because of the prefix issues 557 if not (abspath == self.base[:-1] or abspath.startswith(self.base)): 558 raise errors.PathNotChild(abspath, self.base) 559 pl = len(self.base) 560 return abspath[pl:].strip('/') 561 562 def local_abspath(self, relpath): 563 """Return the absolute path on the local filesystem. 564 565 This function will only be defined for Transports which have a 566 physical local filesystem representation. 567 568 :raises errors.NotLocalUrl: When no local path representation is 569 available. 570 """ 571 raise errors.NotLocalUrl(self.abspath(relpath)) 572 573 def has(self, relpath): 574 """Does the file relpath exist? 575 576 Note that some transports MAY allow querying on directories, but this 577 is not part of the protocol. In other words, the results of 578 t.has("a_directory_name") are undefined. 579 580 :rtype: bool 581 """ 582 raise NotImplementedError(self.has) 583 584 def has_any(self, relpaths): 585 """Return True if any of the paths exist.""" 586 for relpath in relpaths: 587 if self.has(relpath): 588 return True 589 return False 590 591 def iter_files_recursive(self): 592 """Iter the relative paths of files in the transports sub-tree. 593 594 *NOTE*: This only lists *files*, not subdirectories! 595 596 As with other listing functions, only some transports implement this,. 597 you may check via listable() to determine if it will. 598 """ 599 raise errors.TransportNotPossible("This transport has not " 600 "implemented iter_files_recursive " 601 "(but must claim to be listable " 602 "to trigger this error).") 603 604 def get(self, relpath): 605 """Get the file at the given relative path. 606 607 This may fail in a number of ways: 608 - HTTP servers may return content for a directory. (unexpected 609 content failure) 610 - FTP servers may indicate NoSuchFile for a directory. 611 - SFTP servers may give a file handle for a directory that will 612 fail on read(). 613 614 For correct use of the interface, be sure to catch errors.PathError 615 when calling it and catch errors.ReadError when reading from the 616 returned object. 617 618 :param relpath: The relative path to the file 619 :rtype: File-like object. 620 """ 621 raise NotImplementedError(self.get) 622 623 def get_bytes(self, relpath): 624 """Get a raw string of the bytes for a file at the given location. 625 626 :param relpath: The relative path to the file 627 """ 628 f = self.get(relpath) 629 try: 630 return f.read() 631 finally: 632 f.close() 633 634 def get_smart_medium(self): 635 """Return a smart client medium for this transport if possible. 636 637 A smart medium doesn't imply the presence of a smart server: it implies 638 that the smart protocol can be tunnelled via this transport. 639 640 :raises NoSmartMedium: if no smart server medium is available. 641 """ 642 raise errors.NoSmartMedium(self) 643 644 def readv(self, relpath, offsets, adjust_for_latency=False, 645 upper_limit=None): 646 """Get parts of the file at the given relative path. 647 648 :param relpath: The path to read data from. 649 :param offsets: A list of (offset, size) tuples. 650 :param adjust_for_latency: Adjust the requested offsets to accomodate 651 transport latency. This may re-order the offsets, expand them to 652 grab adjacent data when there is likely a high cost to requesting 653 data relative to delivering it. 654 :param upper_limit: When adjust_for_latency is True setting upper_limit 655 allows the caller to tell the transport about the length of the 656 file, so that requests are not issued for ranges beyond the end of 657 the file. This matters because some servers and/or transports error 658 in such a case rather than just satisfying the available ranges. 659 upper_limit should always be provided when adjust_for_latency is 660 True, and should be the size of the file in bytes. 661 :return: A list or generator of (offset, data) tuples 662 """ 663 if adjust_for_latency: 664 # Design note: We may wish to have different algorithms for the 665 # expansion of the offsets per-transport. E.g. for local disk to 666 # use page-aligned expansion. If that is the case consider the 667 # following structure: 668 # - a test that transport.readv uses self._offset_expander or some 669 # similar attribute, to do the expansion 670 # - a test for each transport that it has some known-good offset 671 # expander 672 # - unit tests for each offset expander 673 # - a set of tests for the offset expander interface, giving 674 # baseline behaviour (which the current transport 675 # adjust_for_latency tests could be repurposed to). 676 offsets = self._sort_expand_and_combine(offsets, upper_limit) 677 return self._readv(relpath, offsets) 678 679 def _readv(self, relpath, offsets): 680 """Get parts of the file at the given relative path. 681 682 :param relpath: The path to read. 683 :param offsets: A list of (offset, size) tuples. 684 :return: A list or generator of (offset, data) tuples 685 """ 686 if not offsets: 687 return 688 689 fp = self.get(relpath) 690 return self._seek_and_read(fp, offsets, relpath) 691 692 def _seek_and_read(self, fp, offsets, relpath='<unknown>'): 693 """An implementation of readv that uses fp.seek and fp.read. 694 695 This uses _coalesce_offsets to issue larger reads and fewer seeks. 696 697 :param fp: A file-like object that supports seek() and read(size). 698 Note that implementations are allowed to call .close() on this file 699 handle, so don't trust that you can use it for other work. 700 :param offsets: A list of offsets to be read from the given file. 701 :return: yield (pos, data) tuples for each request 702 """ 703 # We are going to iterate multiple times, we need a list 704 offsets = list(offsets) 705 sorted_offsets = sorted(offsets) 706 707 # turn the list of offsets into a stack 708 offset_stack = iter(offsets) 709 cur_offset_and_size = next(offset_stack) 710 coalesced = self._coalesce_offsets(sorted_offsets, 711 limit=self._max_readv_combine, 712 fudge_factor=self._bytes_to_read_before_seek) 713 714 # Cache the results, but only until they have been fulfilled 715 data_map = {} 716 try: 717 for c_offset in coalesced: 718 # TODO: jam 20060724 it might be faster to not issue seek if 719 # we are already at the right location. This should be 720 # benchmarked. 721 fp.seek(c_offset.start) 722 data = fp.read(c_offset.length) 723 if len(data) < c_offset.length: 724 raise errors.ShortReadvError(relpath, c_offset.start, 725 c_offset.length, actual=len(data)) 726 for suboffset, subsize in c_offset.ranges: 727 key = (c_offset.start + suboffset, subsize) 728 data_map[key] = data[suboffset:suboffset + subsize] 729 730 # Now that we've read some data, see if we can yield anything back 731 while cur_offset_and_size in data_map: 732 this_data = data_map.pop(cur_offset_and_size) 733 this_offset = cur_offset_and_size[0] 734 try: 735 cur_offset_and_size = next(offset_stack) 736 except StopIteration: 737 fp.close() 738 cur_offset_and_size = None 739 yield this_offset, this_data 740 finally: 741 fp.close() 742 743 def _sort_expand_and_combine(self, offsets, upper_limit): 744 """Helper for readv. 745 746 :param offsets: A readv vector - (offset, length) tuples. 747 :param upper_limit: The highest byte offset that may be requested. 748 :return: A readv vector that will read all the regions requested by 749 offsets, in start-to-end order, with no duplicated regions, 750 expanded by the transports recommended page size. 751 """ 752 offsets = sorted(offsets) 753 # short circuit empty requests 754 if len(offsets) == 0: 755 def empty_yielder(): 756 # Quick thunk to stop this function becoming a generator 757 # itself, rather we return a generator that has nothing to 758 # yield. 759 if False: 760 yield None 761 return empty_yielder() 762 # expand by page size at either end 763 maximum_expansion = self.recommended_page_size() 764 new_offsets = [] 765 for offset, length in offsets: 766 expansion = maximum_expansion - length 767 if expansion < 0: 768 # we're asking for more than the minimum read anyway. 769 expansion = 0 770 reduction = expansion // 2 771 new_offset = offset - reduction 772 new_length = length + expansion 773 if new_offset < 0: 774 # don't ask for anything < 0 775 new_offset = 0 776 if (upper_limit is not None and 777 new_offset + new_length > upper_limit): 778 new_length = upper_limit - new_offset 779 new_offsets.append((new_offset, new_length)) 780 # combine the expanded offsets 781 offsets = [] 782 current_offset, current_length = new_offsets[0] 783 current_finish = current_length + current_offset 784 for offset, length in new_offsets[1:]: 785 finish = offset + length 786 if offset > current_finish: 787 # there is a gap, output the current accumulator and start 788 # a new one for the region we're examining. 789 offsets.append((current_offset, current_length)) 790 current_offset = offset 791 current_length = length 792 current_finish = finish 793 continue 794 if finish > current_finish: 795 # extend the current accumulator to the end of the region 796 # we're examining. 797 current_finish = finish 798 current_length = finish - current_offset 799 offsets.append((current_offset, current_length)) 800 return offsets 801 802 @staticmethod 803 def _coalesce_offsets(offsets, limit=0, fudge_factor=0, max_size=0): 804 """Yield coalesced offsets. 805 806 With a long list of neighboring requests, combine them 807 into a single large request, while retaining the original 808 offsets. 809 Turns [(15, 10), (25, 10)] => [(15, 20, [(0, 10), (10, 10)])] 810 Note that overlapping requests are not permitted. (So [(15, 10), (20, 811 10)] will raise a ValueError.) This is because the data we access never 812 overlaps, and it allows callers to trust that we only need any byte of 813 data for 1 request (so nothing needs to be buffered to fulfill a second 814 request.) 815 816 :param offsets: A list of (start, length) pairs 817 :param limit: Only combine a maximum of this many pairs Some transports 818 penalize multiple reads more than others, and sometimes it is 819 better to return early. 820 0 means no limit 821 :param fudge_factor: All transports have some level of 'it is 822 better to read some more data and throw it away rather 823 than seek', so collapse if we are 'close enough' 824 :param max_size: Create coalesced offsets no bigger than this size. 825 When a single offset is bigger than 'max_size', it will keep 826 its size and be alone in the coalesced offset. 827 0 means no maximum size. 828 :return: return a list of _CoalescedOffset objects, which have members 829 for where to start, how much to read, and how to split those chunks 830 back up 831 """ 832 last_end = None 833 cur = _CoalescedOffset(None, None, []) 834 coalesced_offsets = [] 835 836 if max_size <= 0: 837 # 'unlimited', but we actually take this to mean 100MB buffer limit 838 max_size = 100 * 1024 * 1024 839 840 for start, size in offsets: 841 end = start + size 842 if (last_end is not None 843 and start <= last_end + fudge_factor 844 and start >= cur.start 845 and (limit <= 0 or len(cur.ranges) < limit) 846 and (max_size <= 0 or end - cur.start <= max_size)): 847 if start < last_end: 848 raise ValueError('Overlapping range not allowed:' 849 ' last range ended at %s, new one starts at %s' 850 % (last_end, start)) 851 cur.length = end - cur.start 852 cur.ranges.append((start - cur.start, size)) 853 else: 854 if cur.start is not None: 855 coalesced_offsets.append(cur) 856 cur = _CoalescedOffset(start, size, [(0, size)]) 857 last_end = end 858 859 if cur.start is not None: 860 coalesced_offsets.append(cur) 861 return coalesced_offsets 862 863 def put_bytes(self, relpath, raw_bytes, mode=None): 864 """Atomically put the supplied bytes into the given location. 865 866 :param relpath: The location to put the contents, relative to the 867 transport base. 868 :param raw_bytes: A bytestring of data. 869 :param mode: Create the file with the given mode. 870 :return: None 871 """ 872 if not isinstance(raw_bytes, bytes): 873 raise TypeError( 874 'raw_bytes must be a plain string, not %s' % type(raw_bytes)) 875 return self.put_file(relpath, BytesIO(raw_bytes), mode=mode) 876 877 def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None, 878 create_parent_dir=False, 879 dir_mode=None): 880 """Copy the string into the target location. 881 882 This function is not strictly safe to use. See 883 Transport.put_bytes_non_atomic for more information. 884 885 :param relpath: The remote location to put the contents. 886 :param raw_bytes: A string object containing the raw bytes to write 887 into the target file. 888 :param mode: Possible access permissions for new file. 889 None means do not set remote permissions. 890 :param create_parent_dir: If we cannot create the target file because 891 the parent directory does not exist, go ahead and 892 create it, and then try again. 893 :param dir_mode: Possible access permissions for new directories. 894 """ 895 if not isinstance(raw_bytes, bytes): 896 raise TypeError( 897 'raw_bytes must be a plain string, not %s' % type(raw_bytes)) 898 self.put_file_non_atomic(relpath, BytesIO(raw_bytes), mode=mode, 899 create_parent_dir=create_parent_dir, 900 dir_mode=dir_mode) 901 902 def put_file(self, relpath, f, mode=None): 903 """Copy the file-like object into the location. 904 905 :param relpath: Location to put the contents, relative to base. 906 :param f: File-like object. 907 :param mode: The mode for the newly created file, 908 None means just use the default. 909 :return: The length of the file that was written. 910 """ 911 raise NotImplementedError(self.put_file) 912 913 def put_file_non_atomic(self, relpath, f, mode=None, 914 create_parent_dir=False, 915 dir_mode=None): 916 """Copy the file-like object into the target location. 917 918 This function is not strictly safe to use. It is only meant to 919 be used when you already know that the target does not exist. 920 It is not safe, because it will open and truncate the remote 921 file. So there may be a time when the file has invalid contents. 922 923 :param relpath: The remote location to put the contents. 924 :param f: File-like object. 925 :param mode: Possible access permissions for new file. 926 None means do not set remote permissions. 927 :param create_parent_dir: If we cannot create the target file because 928 the parent directory does not exist, go ahead and 929 create it, and then try again. 930 :param dir_mode: Possible access permissions for new directories. 931 """ 932 # Default implementation just does an atomic put. 933 try: 934 return self.put_file(relpath, f, mode=mode) 935 except errors.NoSuchFile: 936 if not create_parent_dir: 937 raise 938 parent_dir = osutils.dirname(relpath) 939 if parent_dir: 940 self.mkdir(parent_dir, mode=dir_mode) 941 return self.put_file(relpath, f, mode=mode) 942 943 def mkdir(self, relpath, mode=None): 944 """Create a directory at the given path.""" 945 raise NotImplementedError(self.mkdir) 946 947 def open_write_stream(self, relpath, mode=None): 948 """Open a writable file stream at relpath. 949 950 A file stream is a file like object with a write() method that accepts 951 bytes to write.. Buffering may occur internally until the stream is 952 closed with stream.close(). Calls to readv or the get_* methods will 953 be synchronised with any internal buffering that may be present. 954 955 :param relpath: The relative path to the file. 956 :param mode: The mode for the newly created file, 957 None means just use the default 958 :return: A FileStream. FileStream objects have two methods, write() and 959 close(). There is no guarantee that data is committed to the file 960 if close() has not been called (even if get() is called on the same 961 path). 962 """ 963 raise NotImplementedError(self.open_write_stream) 964 965 def append_file(self, relpath, f, mode=None): 966 """Append bytes from a file-like object to a file at relpath. 967 968 The file is created if it does not already exist. 969 970 :param f: a file-like object of the bytes to append. 971 :param mode: Unix mode for newly created files. This is not used for 972 existing files. 973 974 :returns: the length of relpath before the content was written to it. 975 """ 976 raise NotImplementedError(self.append_file) 977 978 def append_bytes(self, relpath, data, mode=None): 979 """Append bytes to a file at relpath. 980 981 The file is created if it does not already exist. 982 983 :param relpath: The relative path to the file. 984 :param data: a string of the bytes to append. 985 :param mode: Unix mode for newly created files. This is not used for 986 existing files. 987 988 :returns: the length of relpath before the content was written to it. 989 """ 990 if not isinstance(data, bytes): 991 raise TypeError( 992 'bytes must be a plain string, not %s' % type(data)) 993 return self.append_file(relpath, BytesIO(data), mode=mode) 994 995 def copy(self, rel_from, rel_to): 996 """Copy the item at rel_from to the location at rel_to. 997 998 Override this for efficiency if a specific transport can do it 999 faster than this default implementation. 1000 """ 1001 self.put_file(rel_to, self.get(rel_from)) 1002 1003 def copy_to(self, relpaths, other, mode=None, pb=None): 1004 """Copy a set of entries from self into another Transport. 1005 1006 :param relpaths: A list/generator of entries to be copied. 1007 :param mode: This is the target mode for the newly created files 1008 TODO: This interface needs to be updated so that the target location 1009 can be different from the source location. 1010 """ 1011 # The dummy implementation just does a simple get + put 1012 def copy_entry(path): 1013 other.put_file(path, self.get(path), mode=mode) 1014 1015 return len(self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)) 1016 1017 def copy_tree(self, from_relpath, to_relpath): 1018 """Copy a subtree from one relpath to another. 1019 1020 If a faster implementation is available, specific transports should 1021 implement it. 1022 """ 1023 source = self.clone(from_relpath) 1024 target = self.clone(to_relpath) 1025 1026 # create target directory with the same rwx bits as source. 1027 # use mask to ensure that bits other than rwx are ignored. 1028 stat = self.stat(from_relpath) 1029 target.mkdir('.', stat.st_mode & 0o777) 1030 source.copy_tree_to_transport(target) 1031 1032 def copy_tree_to_transport(self, to_transport): 1033 """Copy a subtree from one transport to another. 1034 1035 self.base is used as the source tree root, and to_transport.base 1036 is used as the target. to_transport.base must exist (and be a 1037 directory). 1038 """ 1039 files = [] 1040 directories = ['.'] 1041 while directories: 1042 dir = directories.pop() 1043 if dir != '.': 1044 to_transport.mkdir(dir) 1045 for path in self.list_dir(dir): 1046 path = dir + '/' + path 1047 stat = self.stat(path) 1048 if S_ISDIR(stat.st_mode): 1049 directories.append(path) 1050 else: 1051 files.append(path) 1052 self.copy_to(files, to_transport) 1053 1054 def rename(self, rel_from, rel_to): 1055 """Rename a file or directory. 1056 1057 This *must* fail if the destination is a nonempty directory - it must 1058 not automatically remove it. It should raise DirectoryNotEmpty, or 1059 some other PathError if the case can't be specifically detected. 1060 1061 If the destination is an empty directory or a file this function may 1062 either fail or succeed, depending on the underlying transport. It 1063 should not attempt to remove the destination if overwriting is not the 1064 native transport behaviour. If at all possible the transport should 1065 ensure that the rename either completes or not, without leaving the 1066 destination deleted and the new file not moved in place. 1067 1068 This is intended mainly for use in implementing LockDir. 1069 """ 1070 # transports may need to override this 1071 raise NotImplementedError(self.rename) 1072 1073 def move(self, rel_from, rel_to): 1074 """Move the item at rel_from to the location at rel_to. 1075 1076 The destination is deleted if possible, even if it's a non-empty 1077 directory tree. 1078 1079 If a transport can directly implement this it is suggested that 1080 it do so for efficiency. 1081 """ 1082 if S_ISDIR(self.stat(rel_from).st_mode): 1083 self.copy_tree(rel_from, rel_to) 1084 self.delete_tree(rel_from) 1085 else: 1086 self.copy(rel_from, rel_to) 1087 self.delete(rel_from) 1088 1089 def delete(self, relpath): 1090 """Delete the item at relpath""" 1091 raise NotImplementedError(self.delete) 1092 1093 def delete_tree(self, relpath): 1094 """Delete an entire tree. This may require a listable transport.""" 1095 subtree = self.clone(relpath) 1096 files = [] 1097 directories = ['.'] 1098 pending_rmdirs = [] 1099 while directories: 1100 dir = directories.pop() 1101 if dir != '.': 1102 pending_rmdirs.append(dir) 1103 for path in subtree.list_dir(dir): 1104 path = dir + '/' + path 1105 stat = subtree.stat(path) 1106 if S_ISDIR(stat.st_mode): 1107 directories.append(path) 1108 else: 1109 files.append(path) 1110 for file in files: 1111 subtree.delete(file) 1112 pending_rmdirs.reverse() 1113 for dir in pending_rmdirs: 1114 subtree.rmdir(dir) 1115 self.rmdir(relpath) 1116 1117 def __repr__(self): 1118 return "<%s.%s url=%s>" % (self.__module__, self.__class__.__name__, self.base) 1119 1120 def stat(self, relpath): 1121 """Return the stat information for a file. 1122 WARNING: This may not be implementable for all protocols, so use 1123 sparingly. 1124 NOTE: This returns an object with fields such as 'st_size'. It MAY 1125 or MAY NOT return the literal result of an os.stat() call, so all 1126 access should be via named fields. 1127 ALSO NOTE: Stats of directories may not be supported on some 1128 transports. 1129 """ 1130 raise NotImplementedError(self.stat) 1131 1132 def rmdir(self, relpath): 1133 """Remove a directory at the given path.""" 1134 raise NotImplementedError 1135 1136 def readlink(self, relpath): 1137 """Return a string representing the path to which the symbolic link points.""" 1138 raise errors.TransportNotPossible( 1139 "Dereferencing symlinks is not supported on %s" % self) 1140 1141 def hardlink(self, source, link_name): 1142 """Create a hardlink pointing to source named link_name.""" 1143 raise errors.TransportNotPossible( 1144 "Hard links are not supported on %s" % self) 1145 1146 def symlink(self, source, link_name): 1147 """Create a symlink pointing to source named link_name.""" 1148 raise errors.TransportNotPossible( 1149 "Symlinks are not supported on %s" % self) 1150 1151 def listable(self): 1152 """Return True if this store supports listing.""" 1153 raise NotImplementedError(self.listable) 1154 1155 def list_dir(self, relpath): 1156 """Return a list of all files at the given location. 1157 WARNING: many transports do not support this, so trying avoid using 1158 it if at all possible. 1159 """ 1160 raise errors.TransportNotPossible("Transport %r has not " 1161 "implemented list_dir " 1162 "(but must claim to be listable " 1163 "to trigger this error)." 1164 % (self)) 1165 1166 def lock_read(self, relpath): 1167 """Lock the given file for shared (read) access. 1168 1169 WARNING: many transports do not support this, so trying avoid using it. 1170 These methods may be removed in the future. 1171 1172 Transports may raise TransportNotPossible if OS-level locks cannot be 1173 taken over this transport. 1174 1175 :return: A lock object, which should contain an unlock() function. 1176 """ 1177 raise errors.TransportNotPossible( 1178 "transport locks not supported on %s" % self) 1179 1180 def lock_write(self, relpath): 1181 """Lock the given file for exclusive (write) access. 1182 1183 WARNING: many transports do not support this, so trying avoid using it. 1184 These methods may be removed in the future. 1185 1186 Transports may raise TransportNotPossible if OS-level locks cannot be 1187 taken over this transport. 1188 1189 :return: A lock object, which should contain an unlock() function. 1190 """ 1191 raise errors.TransportNotPossible( 1192 "transport locks not supported on %s" % self) 1193 1194 def is_readonly(self): 1195 """Return true if this connection cannot be written to.""" 1196 return False 1197 1198 def _can_roundtrip_unix_modebits(self): 1199 """Return true if this transport can store and retrieve unix modebits. 1200 1201 (For example, 0700 to make a directory owner-private.) 1202 1203 Note: most callers will not want to switch on this, but should rather 1204 just try and set permissions and let them be either stored or not. 1205 This is intended mainly for the use of the test suite. 1206 1207 Warning: this is not guaranteed to be accurate as sometimes we can't 1208 be sure: for example with vfat mounted on unix, or a windows sftp 1209 server.""" 1210 # TODO: Perhaps return a e.g. TransportCharacteristics that can answer 1211 # several questions about the transport. 1212 return False 1213 1214 def _reuse_for(self, other_base): 1215 # This is really needed for ConnectedTransport only, but it's easier to 1216 # have Transport refuses to be reused than testing that the reuse 1217 # should be asked to ConnectedTransport only. 1218 return None 1219 1220 def disconnect(self): 1221 # This is really needed for ConnectedTransport only, but it's easier to 1222 # have Transport do nothing than testing that the disconnect should be 1223 # asked to ConnectedTransport only. 1224 pass 1225 1226 def _redirected_to(self, source, target): 1227 """Returns a transport suitable to re-issue a redirected request. 1228 1229 :param source: The source url as returned by the server. 1230 :param target: The target url as returned by the server. 1231 1232 The redirection can be handled only if the relpath involved is not 1233 renamed by the redirection. 1234 1235 :returns: A transport 1236 :raise UnusableRedirect: when redirection can not be provided 1237 """ 1238 # This returns None by default, meaning the transport can't handle the 1239 # redirection. 1240 raise UnusableRedirect( 1241 source, target, "transport does not support redirection") 1242 1243 1244class _SharedConnection(object): 1245 """A connection shared between several transports.""" 1246 1247 def __init__(self, connection=None, credentials=None, base=None): 1248 """Constructor. 1249 1250 :param connection: An opaque object specific to each transport. 1251 1252 :param credentials: An opaque object containing the credentials used to 1253 create the connection. 1254 """ 1255 self.connection = connection 1256 self.credentials = credentials 1257 self.base = base 1258 1259 1260class ConnectedTransport(Transport): 1261 """A transport connected to a remote server. 1262 1263 This class provide the basis to implement transports that need to connect 1264 to a remote server. 1265 1266 Host and credentials are available as private attributes, cloning preserves 1267 them and share the underlying, protocol specific, connection. 1268 """ 1269 1270 def __init__(self, base, _from_transport=None): 1271 """Constructor. 1272 1273 The caller should ensure that _from_transport points at the same host 1274 as the new base. 1275 1276 :param base: transport root URL 1277 1278 :param _from_transport: optional transport to build from. The built 1279 transport will share the connection with this transport. 1280 """ 1281 if not base.endswith('/'): 1282 base += '/' 1283 self._parsed_url = self._split_url(base) 1284 if _from_transport is not None: 1285 # Copy the password as it does not appear in base and will be lost 1286 # otherwise. It can appear in the _split_url above if the user 1287 # provided it on the command line. Otherwise, daughter classes will 1288 # prompt the user for one when appropriate. 1289 self._parsed_url.password = _from_transport._parsed_url.password 1290 self._parsed_url.quoted_password = ( 1291 _from_transport._parsed_url.quoted_password) 1292 1293 base = str(self._parsed_url) 1294 1295 super(ConnectedTransport, self).__init__(base) 1296 if _from_transport is None: 1297 self._shared_connection = _SharedConnection() 1298 else: 1299 self._shared_connection = _from_transport._shared_connection 1300 1301 @property 1302 def _user(self): 1303 return self._parsed_url.user 1304 1305 @property 1306 def _password(self): 1307 return self._parsed_url.password 1308 1309 @property 1310 def _host(self): 1311 return self._parsed_url.host 1312 1313 @property 1314 def _port(self): 1315 return self._parsed_url.port 1316 1317 @property 1318 def _path(self): 1319 return self._parsed_url.path 1320 1321 @property 1322 def _scheme(self): 1323 return self._parsed_url.scheme 1324 1325 def clone(self, offset=None): 1326 """Return a new transport with root at self.base + offset 1327 1328 We leave the daughter classes take advantage of the hint 1329 that it's a cloning not a raw creation. 1330 """ 1331 if offset is None: 1332 return self.__class__(self.base, _from_transport=self) 1333 else: 1334 return self.__class__(self.abspath(offset), _from_transport=self) 1335 1336 @staticmethod 1337 def _split_url(url): 1338 return urlutils.URL.from_string(url) 1339 1340 @staticmethod 1341 def _unsplit_url(scheme, user, password, host, port, path): 1342 """Build the full URL for the given already URL encoded path. 1343 1344 user, password, host and path will be quoted if they contain reserved 1345 chars. 1346 1347 :param scheme: protocol 1348 :param user: login 1349 :param password: associated password 1350 :param host: the server address 1351 :param port: the associated port 1352 :param path: the absolute path on the server 1353 1354 :return: The corresponding URL. 1355 """ 1356 netloc = urlutils.quote(host) 1357 if user is not None: 1358 # Note that we don't put the password back even if we 1359 # have one so that it doesn't get accidentally 1360 # exposed. 1361 netloc = '%s@%s' % (urlutils.quote(user), netloc) 1362 if port is not None: 1363 netloc = '%s:%d' % (netloc, port) 1364 path = urlutils.escape(path) 1365 return urlutils.urlparse.urlunparse((scheme, netloc, path, None, None, None)) 1366 1367 def relpath(self, abspath): 1368 """Return the local path portion from a given absolute path""" 1369 parsed_url = self._split_url(abspath) 1370 error = [] 1371 if parsed_url.scheme != self._parsed_url.scheme: 1372 error.append('scheme mismatch') 1373 if parsed_url.user != self._parsed_url.user: 1374 error.append('user name mismatch') 1375 if parsed_url.host != self._parsed_url.host: 1376 error.append('host mismatch') 1377 if parsed_url.port != self._parsed_url.port: 1378 error.append('port mismatch') 1379 if (not (parsed_url.path == self._parsed_url.path[:-1] or 1380 parsed_url.path.startswith(self._parsed_url.path))): 1381 error.append('path mismatch') 1382 if error: 1383 extra = ', '.join(error) 1384 raise errors.PathNotChild(abspath, self.base, extra=extra) 1385 pl = len(self._parsed_url.path) 1386 return parsed_url.path[pl:].strip('/') 1387 1388 def abspath(self, relpath): 1389 """Return the full url to the given relative path. 1390 1391 :param relpath: the relative path urlencoded 1392 1393 :returns: the Unicode version of the absolute path for relpath. 1394 """ 1395 return str(self._parsed_url.clone(relpath)) 1396 1397 def _remote_path(self, relpath): 1398 """Return the absolute path part of the url to the given relative path. 1399 1400 This is the path that the remote server expect to receive in the 1401 requests, daughter classes should redefine this method if needed and 1402 use the result to build their requests. 1403 1404 :param relpath: the path relative to the transport base urlencoded. 1405 1406 :return: the absolute Unicode path on the server, 1407 """ 1408 return self._parsed_url.clone(relpath).path 1409 1410 def _get_shared_connection(self): 1411 """Get the object shared amongst cloned transports. 1412 1413 This should be used only by classes that needs to extend the sharing 1414 with objects other than transports. 1415 1416 Use _get_connection to get the connection itself. 1417 """ 1418 return self._shared_connection 1419 1420 def _set_connection(self, connection, credentials=None): 1421 """Record a newly created connection with its associated credentials. 1422 1423 Note: To ensure that connection is still shared after a temporary 1424 failure and a new one needs to be created, daughter classes should 1425 always call this method to set the connection and do so each time a new 1426 connection is created. 1427 1428 :param connection: An opaque object representing the connection used by 1429 the daughter class. 1430 1431 :param credentials: An opaque object representing the credentials 1432 needed to create the connection. 1433 """ 1434 self._shared_connection.connection = connection 1435 self._shared_connection.credentials = credentials 1436 for hook in self.hooks["post_connect"]: 1437 hook(self) 1438 1439 def _get_connection(self): 1440 """Returns the transport specific connection object.""" 1441 return self._shared_connection.connection 1442 1443 def _get_credentials(self): 1444 """Returns the credentials used to establish the connection.""" 1445 return self._shared_connection.credentials 1446 1447 def _update_credentials(self, credentials): 1448 """Update the credentials of the current connection. 1449 1450 Some protocols can renegociate the credentials within a connection, 1451 this method allows daughter classes to share updated credentials. 1452 1453 :param credentials: the updated credentials. 1454 """ 1455 # We don't want to call _set_connection here as we are only updating 1456 # the credentials not creating a new connection. 1457 self._shared_connection.credentials = credentials 1458 1459 def _reuse_for(self, other_base): 1460 """Returns a transport sharing the same connection if possible. 1461 1462 Note: we share the connection if the expected credentials are the 1463 same: (host, port, user). Some protocols may disagree and redefine the 1464 criteria in daughter classes. 1465 1466 Note: we don't compare the passwords here because other_base may have 1467 been obtained from an existing transport.base which do not mention the 1468 password. 1469 1470 :param other_base: the URL we want to share the connection with. 1471 1472 :return: A new transport or None if the connection cannot be shared. 1473 """ 1474 try: 1475 parsed_url = self._split_url(other_base) 1476 except urlutils.InvalidURL: 1477 # No hope in trying to reuse an existing transport for an invalid 1478 # URL 1479 return None 1480 1481 transport = None 1482 # Don't compare passwords, they may be absent from other_base or from 1483 # self and they don't carry more information than user anyway. 1484 if (parsed_url.scheme == self._parsed_url.scheme 1485 and parsed_url.user == self._parsed_url.user 1486 and parsed_url.host == self._parsed_url.host 1487 and parsed_url.port == self._parsed_url.port): 1488 path = parsed_url.path 1489 if not path.endswith('/'): 1490 # This normally occurs at __init__ time, but it's easier to do 1491 # it now to avoid creating two transports for the same base. 1492 path += '/' 1493 if self._parsed_url.path == path: 1494 # shortcut, it's really the same transport 1495 return self 1496 # We don't call clone here because the intent is different: we 1497 # build a new transport on a different base (which may be totally 1498 # unrelated) but we share the connection. 1499 transport = self.__class__(other_base, _from_transport=self) 1500 return transport 1501 1502 def disconnect(self): 1503 """Disconnect the transport. 1504 1505 If and when required the transport willl reconnect automatically. 1506 """ 1507 raise NotImplementedError(self.disconnect) 1508 1509 1510def get_transport_from_path(path, possible_transports=None): 1511 """Open a transport for a local path. 1512 1513 :param path: Local path as byte or unicode string 1514 :return: Transport object for path 1515 """ 1516 return get_transport_from_url(urlutils.local_path_to_url(path), 1517 possible_transports) 1518 1519 1520def get_transport_from_url(url, possible_transports=None): 1521 """Open a transport to access a URL. 1522 1523 :param base: a URL 1524 :param transports: optional reusable transports list. If not None, created 1525 transports will be added to the list. 1526 1527 :return: A new transport optionally sharing its connection with one of 1528 possible_transports. 1529 """ 1530 transport = None 1531 if possible_transports is not None: 1532 for t in possible_transports: 1533 t_same_connection = t._reuse_for(url) 1534 if t_same_connection is not None: 1535 # Add only new transports 1536 if t_same_connection not in possible_transports: 1537 possible_transports.append(t_same_connection) 1538 return t_same_connection 1539 1540 last_err = None 1541 for proto, factory_list in transport_list_registry.items(): 1542 if proto is not None and url.startswith(proto): 1543 transport, last_err = _try_transport_factories(url, factory_list) 1544 if transport: 1545 if possible_transports is not None: 1546 if transport in possible_transports: 1547 raise AssertionError() 1548 possible_transports.append(transport) 1549 return transport 1550 if not urlutils.is_url(url): 1551 raise urlutils.InvalidURL(path=url) 1552 raise errors.UnsupportedProtocol(url, last_err) 1553 1554 1555def get_transport(base, possible_transports=None, purpose=None): 1556 """Open a transport to access a URL or directory. 1557 1558 :param base: either a URL or a directory name. 1559 1560 :param transports: optional reusable transports list. If not None, created 1561 transports will be added to the list. 1562 1563 :param purpose: Purpose for which the transport will be used 1564 (e.g. 'read', 'write' or None) 1565 1566 :return: A new transport optionally sharing its connection with one of 1567 possible_transports. 1568 """ 1569 if base is None: 1570 base = '.' 1571 from ..location import location_to_url 1572 return get_transport_from_url( 1573 location_to_url(base, purpose=purpose), 1574 possible_transports) 1575 1576 1577def _try_transport_factories(base, factory_list): 1578 last_err = None 1579 for factory in factory_list: 1580 try: 1581 return factory.get_obj()(base), None 1582 except errors.DependencyNotPresent as e: 1583 mutter("failed to instantiate transport %r for %r: %r" % 1584 (factory, base, e)) 1585 last_err = e 1586 continue 1587 return None, last_err 1588 1589 1590def do_catching_redirections(action, transport, redirected): 1591 """Execute an action with given transport catching redirections. 1592 1593 This is a facility provided for callers needing to follow redirections 1594 silently. The silence is relative: it is the caller responsability to 1595 inform the user about each redirection or only inform the user of a user 1596 via the exception parameter. 1597 1598 :param action: A callable, what the caller want to do while catching 1599 redirections. 1600 :param transport: The initial transport used. 1601 :param redirected: A callable receiving the redirected transport and the 1602 RedirectRequested exception. 1603 1604 :return: Whatever 'action' returns 1605 """ 1606 MAX_REDIRECTIONS = 8 1607 1608 # If a loop occurs, there is little we can do. So we don't try to detect 1609 # them, just getting out if too much redirections occurs. The solution 1610 # is outside: where the loop is defined. 1611 for redirections in range(MAX_REDIRECTIONS): 1612 try: 1613 return action(transport) 1614 except errors.RedirectRequested as e: 1615 redirection_notice = '%s is%s redirected to %s' % ( 1616 e.source, e.permanently, e.target) 1617 transport = redirected(transport, e, redirection_notice) 1618 else: 1619 # Loop exited without resolving redirect ? Either the 1620 # user has kept a very very very old reference or a loop 1621 # occurred in the redirections. Nothing we can cure here: 1622 # tell the user. Note that as the user has been informed 1623 # about each redirection (it is the caller responsibility 1624 # to do that in redirected via the provided 1625 # redirection_notice). The caller may provide more 1626 # information if needed (like what file or directory we 1627 # were trying to act upon when the redirection loop 1628 # occurred). 1629 raise errors.TooManyRedirections 1630 1631 1632class Server(object): 1633 """A Transport Server. 1634 1635 The Server interface provides a server for a given transport type. 1636 """ 1637 1638 def start_server(self): 1639 """Setup the server to service requests.""" 1640 1641 def stop_server(self): 1642 """Remove the server and cleanup any resources it owns.""" 1643 1644 1645# None is the default transport, for things with no url scheme 1646register_transport_proto('file://', 1647 help="Access using the standard filesystem (default)") 1648register_lazy_transport('file://', 'breezy.transport.local', 'LocalTransport') 1649 1650register_transport_proto('sftp://', 1651 help="Access using SFTP (most SSH servers provide SFTP).", 1652 register_netloc=True) 1653register_lazy_transport('sftp://', 'breezy.transport.sftp', 'SFTPTransport') 1654# Decorated http transport 1655register_transport_proto('http+urllib://', 1656 # help="Read-only access of branches exported on the web." 1657 register_netloc=True) 1658register_lazy_transport('http+urllib://', 'breezy.transport.http.urllib', 1659 'HttpTransport') 1660register_transport_proto('https+urllib://', 1661 # help="Read-only access of branches exported on the web using SSL." 1662 register_netloc=True) 1663register_lazy_transport('https+urllib://', 'breezy.transport.http.urllib', 1664 'HttpTransport') 1665# Default http transports (last declared wins (if it can be imported)) 1666register_transport_proto('http://', 1667 help="Read-only access of branches exported on the web.") 1668register_transport_proto('https://', 1669 help="Read-only access of branches exported on the web using SSL.") 1670# The default http implementation is urllib 1671register_lazy_transport('http://', 'breezy.transport.http.urllib', 1672 'HttpTransport') 1673register_lazy_transport('https://', 'breezy.transport.http.urllib', 1674 'HttpTransport') 1675 1676register_transport_proto( 1677 'gio+', help="Access using any GIO supported protocols.") 1678register_lazy_transport( 1679 'gio+', 'breezy.transport.gio_transport', 'GioTransport') 1680 1681 1682register_transport_proto('memory://') 1683register_lazy_transport('memory://', 'breezy.transport.memory', 1684 'MemoryTransport') 1685 1686register_transport_proto('readonly+', 1687 # help="This modifier converts any transport to be readonly." 1688 ) 1689register_lazy_transport('readonly+', 'breezy.transport.readonly', 1690 'ReadonlyTransportDecorator') 1691 1692register_transport_proto('fakenfs+') 1693register_lazy_transport('fakenfs+', 'breezy.transport.fakenfs', 1694 'FakeNFSTransportDecorator') 1695 1696register_transport_proto('log+') 1697register_lazy_transport('log+', 'breezy.transport.log', 1698 'TransportLogDecorator') 1699 1700register_transport_proto('trace+') 1701register_lazy_transport('trace+', 'breezy.transport.trace', 1702 'TransportTraceDecorator') 1703 1704register_transport_proto('unlistable+') 1705register_lazy_transport('unlistable+', 'breezy.transport.unlistable', 1706 'UnlistableTransportDecorator') 1707 1708register_transport_proto('brokenrename+') 1709register_lazy_transport('brokenrename+', 'breezy.transport.brokenrename', 1710 'BrokenRenameTransportDecorator') 1711 1712register_transport_proto('vfat+') 1713register_lazy_transport('vfat+', 1714 'breezy.transport.fakevfat', 1715 'FakeVFATTransportDecorator') 1716 1717register_transport_proto('nosmart+') 1718register_lazy_transport('nosmart+', 'breezy.transport.nosmart', 1719 'NoSmartTransportDecorator') 1720 1721register_transport_proto('bzr://', 1722 help="Fast access using the Bazaar smart server.", 1723 register_netloc=True) 1724 1725register_lazy_transport('bzr://', 'breezy.transport.remote', 1726 'RemoteTCPTransport') 1727register_transport_proto('bzr-v2://', register_netloc=True) 1728 1729register_lazy_transport('bzr-v2://', 'breezy.transport.remote', 1730 'RemoteTCPTransportV2Only') 1731register_transport_proto('bzr+http://', 1732 # help="Fast access using the Bazaar smart server over HTTP." 1733 register_netloc=True) 1734register_lazy_transport('bzr+http://', 'breezy.transport.remote', 1735 'RemoteHTTPTransport') 1736register_transport_proto('bzr+https://', 1737 # help="Fast access using the Bazaar smart server over HTTPS." 1738 register_netloc=True) 1739register_lazy_transport('bzr+https://', 1740 'breezy.transport.remote', 1741 'RemoteHTTPTransport') 1742register_transport_proto('bzr+ssh://', 1743 help="Fast access using the Bazaar smart server over SSH.", 1744 register_netloc=True) 1745register_lazy_transport('bzr+ssh://', 'breezy.transport.remote', 1746 'RemoteSSHTransport') 1747 1748register_transport_proto('ssh:') 1749register_lazy_transport('ssh:', 'breezy.transport.remote', 1750 'HintingSSHTransport') 1751 1752 1753transport_server_registry = registry.Registry() 1754transport_server_registry.register_lazy('bzr', 'breezy.bzr.smart.server', 1755 'serve_bzr', help="The Bazaar smart server protocol over TCP. (default port: 4155)") 1756transport_server_registry.default_key = 'bzr' 1757