1# Copyright (C) 2006-2010 Canonical Ltd 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation; either version 2 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 16 17"""A collection of function for handling URL operations.""" 18 19import os 20import re 21import sys 22 23from urllib import parse as urlparse 24 25from . import ( 26 errors, 27 osutils, 28 ) 29 30from .lazy_import import lazy_import 31lazy_import(globals(), """ 32from posixpath import split as _posix_split 33""") 34 35 36 37class InvalidURL(errors.PathError): 38 39 _fmt = 'Invalid url supplied to transport: "%(path)s"%(extra)s' 40 41 42class InvalidURLJoin(errors.PathError): 43 44 _fmt = "Invalid URL join request: %(reason)s: %(base)r + %(join_args)r" 45 46 def __init__(self, reason, base, join_args): 47 self.reason = reason 48 self.base = base 49 self.join_args = join_args 50 errors.PathError.__init__(self, base, reason) 51 52 53class InvalidRebaseURLs(errors.PathError): 54 55 _fmt = "URLs differ by more than path: %(from_)r and %(to)r" 56 57 def __init__(self, from_, to): 58 self.from_ = from_ 59 self.to = to 60 errors.PathError.__init__( 61 self, from_, 'URLs differ by more than path.') 62 63 64def basename(url, exclude_trailing_slash=True): 65 """Return the last component of a URL. 66 67 :param url: The URL in question 68 :param exclude_trailing_slash: If the url looks like "path/to/foo/" 69 ignore the final slash and return 'foo' rather than '' 70 :return: Just the final component of the URL. This can return '' 71 if you don't exclude_trailing_slash, or if you are at the 72 root of the URL. 73 """ 74 return split(url, exclude_trailing_slash=exclude_trailing_slash)[1] 75 76 77def dirname(url, exclude_trailing_slash=True): 78 """Return the parent directory of the given path. 79 80 :param url: Relative or absolute URL 81 :param exclude_trailing_slash: Remove a final slash 82 (treat http://host/foo/ as http://host/foo, but 83 http://host/ stays http://host/) 84 :return: Everything in the URL except the last path chunk 85 """ 86 # TODO: jam 20060502 This was named dirname to be consistent 87 # with the os functions, but maybe "parent" would be better 88 return split(url, exclude_trailing_slash=exclude_trailing_slash)[0] 89 90 91quote_from_bytes = urlparse.quote_from_bytes 92quote = urlparse.quote 93unquote_to_bytes = urlparse.unquote_to_bytes 94unquote = urlparse.unquote 95 96 97def escape(relpath, safe='/~'): 98 """Escape relpath to be a valid url.""" 99 return quote(relpath, safe=safe) 100 101 102def file_relpath(base, path): 103 """Compute just the relative sub-portion of a url 104 105 This assumes that both paths are already fully specified file:// URLs. 106 """ 107 if len(base) < MIN_ABS_FILEURL_LENGTH: 108 raise ValueError('Length of base (%r) must equal or' 109 ' exceed the platform minimum url length (which is %d)' % 110 (base, MIN_ABS_FILEURL_LENGTH)) 111 base = osutils.normpath(local_path_from_url(base)) 112 path = osutils.normpath(local_path_from_url(path)) 113 return escape(osutils.relpath(base, path)) 114 115 116def _find_scheme_and_separator(url): 117 """Find the scheme separator (://) and the first path separator 118 119 This is just a helper functions for other path utilities. 120 It could probably be replaced by urlparse 121 """ 122 m = _url_scheme_re.match(url) 123 if not m: 124 return None, None 125 126 scheme = m.group('scheme') 127 path = m.group('path') 128 129 # Find the path separating slash 130 # (first slash after the ://) 131 first_path_slash = path.find('/') 132 if first_path_slash == -1: 133 return len(scheme), None 134 return len(scheme), first_path_slash + m.start('path') 135 136 137def is_url(url): 138 """Tests whether a URL is in actual fact a URL.""" 139 return _url_scheme_re.match(url) is not None 140 141 142def join(base, *args): 143 """Create a URL by joining sections. 144 145 This will normalize '..', assuming that paths are absolute 146 (it assumes no symlinks in either path) 147 148 If any of *args is an absolute URL, it will be treated correctly. 149 Example: 150 join('http://foo', 'http://bar') => 'http://bar' 151 join('http://foo', 'bar') => 'http://foo/bar' 152 join('http://foo', 'bar', '../baz') => 'http://foo/baz' 153 """ 154 if not args: 155 return base 156 scheme_end, path_start = _find_scheme_and_separator(base) 157 if scheme_end is None and path_start is None: 158 path_start = 0 159 elif path_start is None: 160 path_start = len(base) 161 path = base[path_start:] 162 for arg in args: 163 arg_scheme_end, arg_path_start = _find_scheme_and_separator(arg) 164 if arg_scheme_end is None and arg_path_start is None: 165 arg_path_start = 0 166 elif arg_path_start is None: 167 arg_path_start = len(arg) 168 if arg_scheme_end is not None: 169 base = arg 170 path = arg[arg_path_start:] 171 scheme_end = arg_scheme_end 172 path_start = arg_path_start 173 else: 174 path = joinpath(path, arg) 175 return base[:path_start] + path 176 177 178def joinpath(base, *args): 179 """Join URL path segments to a URL path segment. 180 181 This is somewhat like osutils.joinpath, but intended for URLs. 182 183 XXX: this duplicates some normalisation logic, and also duplicates a lot of 184 path handling logic that already exists in some Transport implementations. 185 We really should try to have exactly one place in the code base responsible 186 for combining paths of URLs. 187 """ 188 path = base.split('/') 189 if len(path) > 1 and path[-1] == '': 190 # If the path ends in a trailing /, remove it. 191 path.pop() 192 for arg in args: 193 if arg.startswith('/'): 194 path = [] 195 for chunk in arg.split('/'): 196 if chunk == '.': 197 continue 198 elif chunk == '..': 199 if path == ['']: 200 raise InvalidURLJoin('Cannot go above root', 201 base, args) 202 path.pop() 203 else: 204 path.append(chunk) 205 if path == ['']: 206 return '/' 207 else: 208 return '/'.join(path) 209 210 211# jam 20060502 Sorted to 'l' because the final target is 'local_path_from_url' 212def _posix_local_path_from_url(url): 213 """Convert a url like file:///path/to/foo into /path/to/foo""" 214 url = strip_segment_parameters(url) 215 file_localhost_prefix = 'file://localhost/' 216 if url.startswith(file_localhost_prefix): 217 path = url[len(file_localhost_prefix) - 1:] 218 elif not url.startswith('file:///'): 219 raise InvalidURL( 220 url, 'local urls must start with file:/// or file://localhost/') 221 else: 222 path = url[len('file://'):] 223 # We only strip off 2 slashes 224 return unescape(path) 225 226 227def _posix_local_path_to_url(path): 228 """Convert a local path like ./foo into a URL like file:///path/to/foo 229 230 This also handles transforming escaping unicode characters, etc. 231 """ 232 # importing directly from posixpath allows us to test this 233 # on non-posix platforms 234 return 'file://' + escape(osutils._posix_abspath(path)) 235 236 237def _win32_local_path_from_url(url): 238 """Convert a url like file:///C:/path/to/foo into C:/path/to/foo""" 239 if not url.startswith('file://'): 240 raise InvalidURL(url, 'local urls must start with file:///, ' 241 'UNC path urls must start with file://') 242 url = strip_segment_parameters(url) 243 # We strip off all 3 slashes 244 win32_url = url[len('file:'):] 245 # check for UNC path: //HOST/path 246 if not win32_url.startswith('///'): 247 if (win32_url[2] == '/' 248 or win32_url[3] in '|:'): 249 raise InvalidURL(url, 'Win32 UNC path urls' 250 ' have form file://HOST/path') 251 return unescape(win32_url) 252 253 # allow empty paths so we can serve all roots 254 if win32_url == '///': 255 return '/' 256 257 # usual local path with drive letter 258 if (len(win32_url) < 6 259 or win32_url[3] not in ('abcdefghijklmnopqrstuvwxyz' 260 'ABCDEFGHIJKLMNOPQRSTUVWXYZ') or 261 win32_url[4] not in '|:' 262 or win32_url[5] != '/'): 263 raise InvalidURL(url, 'Win32 file urls start with' 264 ' file:///x:/, where x is a valid drive letter') 265 return win32_url[3].upper() + u':' + unescape(win32_url[5:]) 266 267 268def _win32_local_path_to_url(path): 269 """Convert a local path like ./foo into a URL like file:///C:/path/to/foo 270 271 This also handles transforming escaping unicode characters, etc. 272 """ 273 # importing directly from ntpath allows us to test this 274 # on non-win32 platform 275 # FIXME: It turns out that on nt, ntpath.abspath uses nt._getfullpathname 276 # which actually strips trailing space characters. 277 # The worst part is that on linux ntpath.abspath has different 278 # semantics, since 'nt' is not an available module. 279 if path == '/': 280 return 'file:///' 281 282 win32_path = osutils._win32_abspath(path) 283 # check for UNC path \\HOST\path 284 if win32_path.startswith('//'): 285 return 'file:' + escape(win32_path) 286 return ('file:///' + str(win32_path[0].upper()) + ':' + 287 escape(win32_path[2:])) 288 289 290local_path_to_url = _posix_local_path_to_url 291local_path_from_url = _posix_local_path_from_url 292MIN_ABS_FILEURL_LENGTH = len('file:///') 293WIN32_MIN_ABS_FILEURL_LENGTH = len('file:///C:/') 294 295if sys.platform == 'win32': 296 local_path_to_url = _win32_local_path_to_url 297 local_path_from_url = _win32_local_path_from_url 298 299 MIN_ABS_FILEURL_LENGTH = WIN32_MIN_ABS_FILEURL_LENGTH 300 301 302_url_scheme_re = re.compile('^(?P<scheme>[^:/]{2,}):(//)?(?P<path>.*)$') 303_url_hex_escapes_re = re.compile('(%[0-9a-fA-F]{2})') 304 305 306def _unescape_safe_chars(matchobj): 307 """re.sub callback to convert hex-escapes to plain characters (if safe). 308 309 e.g. '%7E' will be converted to '~'. 310 """ 311 hex_digits = matchobj.group(0)[1:] 312 char = chr(int(hex_digits, 16)) 313 if char in _url_dont_escape_characters: 314 return char 315 else: 316 return matchobj.group(0).upper() 317 318 319def normalize_url(url): 320 """Make sure that a path string is in fully normalized URL form. 321 322 This handles URLs which have unicode characters, spaces, 323 special characters, etc. 324 325 It has two basic modes of operation, depending on whether the 326 supplied string starts with a url specifier (scheme://) or not. 327 If it does not have a specifier it is considered a local path, 328 and will be converted into a file:/// url. Non-ascii characters 329 will be encoded using utf-8. 330 If it does have a url specifier, it will be treated as a "hybrid" 331 URL. Basically, a URL that should have URL special characters already 332 escaped (like +?&# etc), but may have unicode characters, etc 333 which would not be valid in a real URL. 334 335 :param url: Either a hybrid URL or a local path 336 :return: A normalized URL which only includes 7-bit ASCII characters. 337 """ 338 scheme_end, path_start = _find_scheme_and_separator(url) 339 if scheme_end is None: 340 return local_path_to_url(url) 341 prefix = url[:path_start] 342 path = url[path_start:] 343 if not isinstance(url, str): 344 for c in url: 345 if c not in _url_safe_characters: 346 raise InvalidURL(url, 'URLs can only contain specific' 347 ' safe characters (not %r)' % c) 348 path = _url_hex_escapes_re.sub(_unescape_safe_chars, path) 349 return str(prefix + ''.join(path)) 350 351 # We have a unicode (hybrid) url 352 path_chars = list(path) 353 354 for i in range(len(path_chars)): 355 if path_chars[i] not in _url_safe_characters: 356 path_chars[i] = ''.join( 357 ['%%%02X' % c for c in bytearray(path_chars[i].encode('utf-8'))]) 358 path = ''.join(path_chars) 359 path = _url_hex_escapes_re.sub(_unescape_safe_chars, path) 360 return str(prefix + path) 361 362 363def relative_url(base, other): 364 """Return a path to other from base. 365 366 If other is unrelated to base, return other. Else return a relative path. 367 This assumes no symlinks as part of the url. 368 """ 369 dummy, base_first_slash = _find_scheme_and_separator(base) 370 if base_first_slash is None: 371 return other 372 373 dummy, other_first_slash = _find_scheme_and_separator(other) 374 if other_first_slash is None: 375 return other 376 377 # this takes care of differing schemes or hosts 378 base_scheme = base[:base_first_slash] 379 other_scheme = other[:other_first_slash] 380 if base_scheme != other_scheme: 381 return other 382 elif sys.platform == 'win32' and base_scheme == 'file://': 383 base_drive = base[base_first_slash + 1:base_first_slash + 3] 384 other_drive = other[other_first_slash + 1:other_first_slash + 3] 385 if base_drive != other_drive: 386 return other 387 388 base_path = base[base_first_slash + 1:] 389 other_path = other[other_first_slash + 1:] 390 391 if base_path.endswith('/'): 392 base_path = base_path[:-1] 393 394 base_sections = base_path.split('/') 395 other_sections = other_path.split('/') 396 397 if base_sections == ['']: 398 base_sections = [] 399 if other_sections == ['']: 400 other_sections = [] 401 402 output_sections = [] 403 for b, o in zip(base_sections, other_sections): 404 if b != o: 405 break 406 output_sections.append(b) 407 408 match_len = len(output_sections) 409 output_sections = ['..' for x in base_sections[match_len:]] 410 output_sections.extend(other_sections[match_len:]) 411 412 return "/".join(output_sections) or "." 413 414 415def _win32_extract_drive_letter(url_base, path): 416 """On win32 the drive letter needs to be added to the url base.""" 417 # Strip off the drive letter 418 # path is currently /C:/foo 419 if len(path) < 4 or path[2] not in ':|' or path[3] != '/': 420 raise InvalidURL(url_base + path, 421 'win32 file:/// paths need a drive letter') 422 url_base += path[0:3] # file:// + /C: 423 path = path[3:] # /foo 424 return url_base, path 425 426 427def split(url, exclude_trailing_slash=True): 428 """Split a URL into its parent directory and a child directory. 429 430 :param url: A relative or absolute URL 431 :param exclude_trailing_slash: Strip off a final '/' if it is part 432 of the path (but not if it is part of the protocol specification) 433 434 :return: (parent_url, child_dir). child_dir may be the empty string if 435 we're at the root. 436 """ 437 scheme_loc, first_path_slash = _find_scheme_and_separator(url) 438 439 if first_path_slash is None: 440 # We have either a relative path, or no separating slash 441 if scheme_loc is None: 442 # Relative path 443 if exclude_trailing_slash and url.endswith('/'): 444 url = url[:-1] 445 return _posix_split(url) 446 else: 447 # Scheme with no path 448 return url, '' 449 450 # We have a fully defined path 451 url_base = url[:first_path_slash] # http://host, file:// 452 path = url[first_path_slash:] # /file/foo 453 454 if sys.platform == 'win32' and url.startswith('file:///'): 455 # Strip off the drive letter 456 # url_base is currently file:// 457 # path is currently /C:/foo 458 url_base, path = _win32_extract_drive_letter(url_base, path) 459 # now it should be file:///C: and /foo 460 461 if exclude_trailing_slash and len(path) > 1 and path.endswith('/'): 462 path = path[:-1] 463 head, tail = _posix_split(path) 464 return url_base + head, tail 465 466 467def split_segment_parameters_raw(url): 468 """Split the subsegment of the last segment of a URL. 469 470 :param url: A relative or absolute URL 471 :return: (url, subsegments) 472 """ 473 # GZ 2011-11-18: Dodgy removing the terminal slash like this, function 474 # operates on urls not url+segments, and Transport classes 475 # should not be blindly adding slashes in the first place. 476 lurl = strip_trailing_slash(url) 477 # Segments begin at first comma after last forward slash, if one exists 478 segment_start = lurl.find(",", lurl.rfind("/") + 1) 479 if segment_start == -1: 480 return (url, []) 481 return (lurl[:segment_start], 482 [str(s) for s in lurl[segment_start + 1:].split(",")]) 483 484 485def split_segment_parameters(url): 486 """Split the segment parameters of the last segment of a URL. 487 488 :param url: A relative or absolute URL 489 :return: (url, segment_parameters) 490 """ 491 (base_url, subsegments) = split_segment_parameters_raw(url) 492 parameters = {} 493 for subsegment in subsegments: 494 try: 495 (key, value) = subsegment.split("=", 1) 496 except ValueError: 497 raise InvalidURL(url, "missing = in subsegment") 498 if not isinstance(key, str): 499 raise TypeError(key) 500 if not isinstance(value, str): 501 raise TypeError(value) 502 parameters[key] = value 503 return (base_url, parameters) 504 505 506def strip_segment_parameters(url): 507 """Strip the segment parameters from a URL. 508 509 :param url: A relative or absolute URL 510 :return: url 511 """ 512 base_url, subsegments = split_segment_parameters_raw(url) 513 return base_url 514 515 516def join_segment_parameters_raw(base, *subsegments): 517 """Create a new URL by adding subsegments to an existing one. 518 519 This adds the specified subsegments to the last path in the specified 520 base URL. The subsegments should be bytestrings. 521 522 :note: You probably want to use join_segment_parameters instead. 523 """ 524 if not subsegments: 525 return base 526 for subsegment in subsegments: 527 if not isinstance(subsegment, str): 528 raise TypeError("Subsegment %r is not a bytestring" % subsegment) 529 if "," in subsegment: 530 raise InvalidURLJoin(", exists in subsegments", 531 base, subsegments) 532 return ",".join((base,) + subsegments) 533 534 535def join_segment_parameters(url, parameters): 536 """Create a new URL by adding segment parameters to an existing one. 537 538 The parameters of the last segment in the URL will be updated; if a 539 parameter with the same key already exists it will be overwritten. 540 541 :param url: A URL, as string 542 :param parameters: Dictionary of parameters, keys and values as bytestrings 543 """ 544 (base, existing_parameters) = split_segment_parameters(url) 545 new_parameters = {} 546 new_parameters.update(existing_parameters) 547 for key, value in parameters.items(): 548 if not isinstance(key, str): 549 raise TypeError("parameter key %r is not a str" % key) 550 if not isinstance(value, str): 551 raise TypeError("parameter value %r for %r is not a str" % 552 (value, key)) 553 if "=" in key: 554 raise InvalidURLJoin("= exists in parameter key", url, 555 parameters) 556 new_parameters[key] = value 557 return join_segment_parameters_raw( 558 base, *["%s=%s" % item for item in sorted(new_parameters.items())]) 559 560 561def _win32_strip_local_trailing_slash(url): 562 """Strip slashes after the drive letter""" 563 if len(url) > WIN32_MIN_ABS_FILEURL_LENGTH: 564 return url[:-1] 565 else: 566 return url 567 568 569def strip_trailing_slash(url): 570 """Strip trailing slash, except for root paths. 571 572 The definition of 'root path' is platform-dependent. 573 This assumes that all URLs are valid netloc urls, such that they 574 form: 575 scheme://host/path 576 It searches for ://, and then refuses to remove the next '/'. 577 It can also handle relative paths 578 Examples: 579 path/to/foo => path/to/foo 580 path/to/foo/ => path/to/foo 581 http://host/path/ => http://host/path 582 http://host/path => http://host/path 583 http://host/ => http://host/ 584 file:/// => file:/// 585 file:///foo/ => file:///foo 586 # This is unique on win32 platforms, and is the only URL 587 # format which does it differently. 588 file:///c|/ => file:///c:/ 589 """ 590 if not url.endswith('/'): 591 # Nothing to do 592 return url 593 if sys.platform == 'win32' and url.startswith('file://'): 594 return _win32_strip_local_trailing_slash(url) 595 596 scheme_loc, first_path_slash = _find_scheme_and_separator(url) 597 if scheme_loc is None: 598 # This is a relative path, as it has no scheme 599 # so just chop off the last character 600 return url[:-1] 601 602 if first_path_slash is None or first_path_slash == len(url) - 1: 603 # Don't chop off anything if the only slash is the path 604 # separating slash 605 return url 606 607 return url[:-1] 608 609 610def unescape(url): 611 """Unescape relpath from url format. 612 613 This returns a Unicode path from a URL 614 """ 615 # jam 20060427 URLs are supposed to be ASCII only strings 616 # If they are passed in as unicode, unquote 617 # will return a UNICODE string, which actually contains 618 # utf-8 bytes. So we have to ensure that they are 619 # plain ASCII strings, or the final .decode will 620 # try to encode the UNICODE => ASCII, and then decode 621 # it into utf-8. 622 623 if isinstance(url, str): 624 try: 625 url.encode("ascii") 626 except UnicodeError as e: 627 raise InvalidURL( 628 url, 'URL was not a plain ASCII url: %s' % (e,)) 629 return urlparse.unquote(url) 630 631 632# These are characters that if escaped, should stay that way 633_no_decode_chars = ';/?:@&=+$,#' 634_no_decode_ords = [ord(c) for c in _no_decode_chars] 635_no_decode_hex = (['%02x' % o for o in _no_decode_ords] 636 + ['%02X' % o for o in _no_decode_ords]) 637_hex_display_map = dict(([('%02x' % o, bytes([o])) for o in range(256)] 638 + [('%02X' % o, bytes([o])) for o in range(256)])) 639# These entries get mapped to themselves 640_hex_display_map.update((hex, b'%' + hex.encode('ascii')) 641 for hex in _no_decode_hex) 642 643# These characters shouldn't be percent-encoded, and it's always safe to 644# unencode them if they are. 645_url_dont_escape_characters = set( 646 "abcdefghijklmnopqrstuvwxyz" # Lowercase alpha 647 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" # Uppercase alpha 648 "0123456789" # Numbers 649 "-._~" # Unreserved characters 650) 651 652# These characters should not be escaped 653_url_safe_characters = set( 654 "abcdefghijklmnopqrstuvwxyz" # Lowercase alpha 655 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" # Uppercase alpha 656 "0123456789" # Numbers 657 "_.-!~*'()" # Unreserved characters 658 "/;?:@&=+$," # Reserved characters 659 "%#" # Extra reserved characters 660) 661 662 663def _unescape_segment_for_display(segment, encoding): 664 """Unescape a segment for display. 665 666 Helper for unescape_for_display 667 668 :param url: A 7-bit ASCII URL 669 :param encoding: The final output encoding 670 671 :return: A unicode string which can be safely encoded into the 672 specified encoding. 673 """ 674 escaped_chunks = segment.split('%') 675 escaped_chunks[0] = escaped_chunks[0].encode('utf-8') 676 for j in range(1, len(escaped_chunks)): 677 item = escaped_chunks[j] 678 try: 679 escaped_chunks[j] = _hex_display_map[item[:2]] 680 except KeyError: 681 # Put back the percent symbol 682 escaped_chunks[j] = b'%' + (item[:2].encode('utf-8')) 683 except UnicodeDecodeError: 684 escaped_chunks[j] = chr(int(item[:2], 16)).encode('utf-8') 685 escaped_chunks[j] += (item[2:].encode('utf-8')) 686 unescaped = b''.join(escaped_chunks) 687 try: 688 decoded = unescaped.decode('utf-8') 689 except UnicodeDecodeError: 690 # If this path segment cannot be properly utf-8 decoded 691 # after doing unescaping we will just leave it alone 692 return segment 693 else: 694 try: 695 decoded.encode(encoding) 696 except UnicodeEncodeError: 697 # If this chunk cannot be encoded in the local 698 # encoding, then we should leave it alone 699 return segment 700 else: 701 # Otherwise take the url decoded one 702 return decoded 703 704 705def unescape_for_display(url, encoding): 706 """Decode what you can for a URL, so that we get a nice looking path. 707 708 This will turn file:// urls into local paths, and try to decode 709 any portions of a http:// style url that it can. 710 711 Any sections of the URL which can't be represented in the encoding or 712 need to stay as escapes are left alone. 713 714 :param url: A 7-bit ASCII URL 715 :param encoding: The final output encoding 716 717 :return: A unicode string which can be safely encoded into the 718 specified encoding. 719 """ 720 if encoding is None: 721 raise ValueError('you cannot specify None for the display encoding') 722 if url.startswith('file://'): 723 try: 724 path = local_path_from_url(url) 725 path.encode(encoding) 726 return path 727 except UnicodeError: 728 return url 729 730 # Split into sections to try to decode utf-8 731 res = url.split('/') 732 for i in range(1, len(res)): 733 res[i] = _unescape_segment_for_display(res[i], encoding) 734 return u'/'.join(res) 735 736 737def derive_to_location(from_location): 738 """Derive a TO_LOCATION given a FROM_LOCATION. 739 740 The normal case is a FROM_LOCATION of http://foo/bar => bar. 741 The Right Thing for some logical destinations may differ though 742 because no / may be present at all. In that case, the result is 743 the full name without the scheme indicator, e.g. lp:foo-bar => foo-bar. 744 This latter case also applies when a Windows drive 745 is used without a path, e.g. c:foo-bar => foo-bar. 746 If no /, path separator or : is found, the from_location is returned. 747 """ 748 from_location = strip_segment_parameters(from_location) 749 if from_location.find("/") >= 0 or from_location.find(os.sep) >= 0: 750 return os.path.basename(from_location.rstrip("/\\")) 751 else: 752 sep = from_location.find(":") 753 if sep > 0: 754 return from_location[sep + 1:] 755 else: 756 return from_location 757 758 759def _is_absolute(url): 760 return (osutils.pathjoin('/foo', url) == url) 761 762 763def rebase_url(url, old_base, new_base): 764 """Convert a relative path from an old base URL to a new base URL. 765 766 The result will be a relative path. 767 Absolute paths and full URLs are returned unaltered. 768 """ 769 scheme, separator = _find_scheme_and_separator(url) 770 if scheme is not None: 771 return url 772 if _is_absolute(url): 773 return url 774 old_parsed = urlparse.urlparse(old_base) 775 new_parsed = urlparse.urlparse(new_base) 776 if (old_parsed[:2]) != (new_parsed[:2]): 777 raise InvalidRebaseURLs(old_base, new_base) 778 return determine_relative_path(new_parsed[2], 779 join(old_parsed[2], url)) 780 781 782def determine_relative_path(from_path, to_path): 783 """Determine a relative path from from_path to to_path.""" 784 from_segments = osutils.splitpath(from_path) 785 to_segments = osutils.splitpath(to_path) 786 count = -1 787 for count, (from_element, to_element) in enumerate(zip(from_segments, 788 to_segments)): 789 if from_element != to_element: 790 break 791 else: 792 count += 1 793 unique_from = from_segments[count:] 794 unique_to = to_segments[count:] 795 segments = (['..'] * len(unique_from) + unique_to) 796 if len(segments) == 0: 797 return '.' 798 return osutils.pathjoin(*segments) 799 800 801class URL(object): 802 """Parsed URL.""" 803 804 def __init__(self, scheme, quoted_user, quoted_password, quoted_host, 805 port, quoted_path): 806 self.scheme = scheme 807 self.quoted_host = quoted_host 808 self.host = unquote(self.quoted_host) 809 self.quoted_user = quoted_user 810 if self.quoted_user is not None: 811 self.user = unquote(self.quoted_user) 812 else: 813 self.user = None 814 self.quoted_password = quoted_password 815 if self.quoted_password is not None: 816 self.password = unquote(self.quoted_password) 817 else: 818 self.password = None 819 self.port = port 820 self.quoted_path = _url_hex_escapes_re.sub( 821 _unescape_safe_chars, quoted_path) 822 self.path = unquote(self.quoted_path) 823 824 def __eq__(self, other): 825 return (isinstance(other, self.__class__) and 826 self.scheme == other.scheme and 827 self.host == other.host and 828 self.user == other.user and 829 self.password == other.password and 830 self.path == other.path) 831 832 def __repr__(self): 833 return "<%s(%r, %r, %r, %r, %r, %r)>" % ( 834 self.__class__.__name__, 835 self.scheme, self.quoted_user, self.quoted_password, 836 self.quoted_host, self.port, self.quoted_path) 837 838 @classmethod 839 def from_string(cls, url): 840 """Create a URL object from a string. 841 842 :param url: URL as bytestring 843 """ 844 # GZ 2017-06-09: Actually validate ascii-ness 845 # pad.lv/1696545: For the moment, accept both native strings and 846 # unicode. 847 if isinstance(url, str): 848 pass 849 elif isinstance(url, str): 850 try: 851 url = url.encode() 852 except UnicodeEncodeError: 853 raise InvalidURL(url) 854 else: 855 raise InvalidURL(url) 856 (scheme, netloc, path, params, 857 query, fragment) = urlparse.urlparse(url, allow_fragments=False) 858 user = password = host = port = None 859 if '@' in netloc: 860 user, host = netloc.rsplit('@', 1) 861 if ':' in user: 862 user, password = user.split(':', 1) 863 else: 864 host = netloc 865 866 if ':' in host and not (host[0] == '[' and host[-1] == ']'): 867 # there *is* port 868 host, port = host.rsplit(':', 1) 869 if port: 870 try: 871 port = int(port) 872 except ValueError: 873 raise InvalidURL('invalid port number %s in url:\n%s' % 874 (port, url)) 875 else: 876 port = None 877 if host != "" and host[0] == '[' and host[-1] == ']': # IPv6 878 host = host[1:-1] 879 880 return cls(scheme, user, password, host, port, path) 881 882 def __str__(self): 883 netloc = self.quoted_host 884 if ":" in netloc: 885 netloc = "[%s]" % netloc 886 if self.quoted_user is not None: 887 # Note that we don't put the password back even if we 888 # have one so that it doesn't get accidentally 889 # exposed. 890 netloc = '%s@%s' % (self.quoted_user, netloc) 891 if self.port is not None: 892 netloc = '%s:%d' % (netloc, self.port) 893 return urlparse.urlunparse( 894 (self.scheme, netloc, self.quoted_path, None, None, None)) 895 896 @staticmethod 897 def _combine_paths(base_path, relpath): 898 """Transform a Transport-relative path to a remote absolute path. 899 900 This does not handle substitution of ~ but does handle '..' and '.' 901 components. 902 903 Examples:: 904 905 t._combine_paths('/home/sarah', 'project/foo') 906 => '/home/sarah/project/foo' 907 t._combine_paths('/home/sarah', '../../etc') 908 => '/etc' 909 t._combine_paths('/home/sarah', '/etc') 910 => '/etc' 911 912 :param base_path: base path 913 :param relpath: relative url string for relative part of remote path. 914 :return: urlencoded string for final path. 915 """ 916 # pad.lv/1696545: For the moment, accept both native strings and 917 # unicode. 918 if isinstance(relpath, str): 919 pass 920 elif isinstance(relpath, str): 921 try: 922 relpath = relpath.encode() 923 except UnicodeEncodeError: 924 raise InvalidURL(relpath) 925 else: 926 raise InvalidURL(relpath) 927 relpath = _url_hex_escapes_re.sub(_unescape_safe_chars, relpath) 928 if relpath.startswith('/'): 929 base_parts = [] 930 else: 931 base_parts = base_path.split('/') 932 if len(base_parts) > 0 and base_parts[-1] == '': 933 base_parts = base_parts[:-1] 934 for p in relpath.split('/'): 935 if p == '..': 936 if len(base_parts) == 0: 937 # In most filesystems, a request for the parent 938 # of root, just returns root. 939 continue 940 base_parts.pop() 941 elif p == '.': 942 continue # No-op 943 elif p != '': 944 base_parts.append(p) 945 path = '/'.join(base_parts) 946 if not path.startswith('/'): 947 path = '/' + path 948 return path 949 950 def clone(self, offset=None): 951 """Return a new URL for a path relative to this URL. 952 953 :param offset: A relative path, already urlencoded 954 :return: `URL` instance 955 """ 956 if offset is not None: 957 relative = unescape(offset) 958 path = self._combine_paths(self.path, relative) 959 path = quote(path, safe="/~") 960 else: 961 path = self.quoted_path 962 return self.__class__(self.scheme, self.quoted_user, 963 self.quoted_password, self.quoted_host, self.port, 964 path) 965 966 967def parse_url(url): 968 """Extract the server address, the credentials and the path from the url. 969 970 user, password, host and path should be quoted if they contain reserved 971 chars. 972 973 :param url: an quoted url 974 :return: (scheme, user, password, host, port, path) tuple, all fields 975 are unquoted. 976 """ 977 parsed_url = URL.from_string(url) 978 return (parsed_url.scheme, parsed_url.user, parsed_url.password, 979 parsed_url.host, parsed_url.port, parsed_url.path) 980