1# Copyright (C) 2005-2012, 2015, 2016, 2017 Canonical Ltd 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation; either version 2 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 16 17"""Tests for HTTP implementations. 18 19This module defines a load_tests() method that parametrize tests classes for 20transport implementation, http protocol versions and authentication schemes. 21""" 22 23# TODO: Should be renamed to breezy.transport.http.tests? 24# TODO: What about renaming to breezy.tests.transport.http ? 25 26from http.client import UnknownProtocol, parse_headers 27from http.server import SimpleHTTPRequestHandler 28import io 29import socket 30import sys 31import threading 32 33import breezy 34from .. import ( 35 config, 36 controldir, 37 debug, 38 errors, 39 osutils, 40 tests, 41 trace, 42 transport, 43 ui, 44 urlutils, 45 ) 46from ..bzr import ( 47 remote as _mod_remote, 48 ) 49from . import ( 50 features, 51 http_server, 52 http_utils, 53 test_server, 54 ) 55from .scenarios import ( 56 load_tests_apply_scenarios, 57 multiply_scenarios, 58 ) 59from ..transport import ( 60 remote, 61 ) 62from ..transport.http import urllib 63from ..transport.http.urllib import ( 64 AbstractAuthHandler, 65 BasicAuthHandler, 66 HttpTransport, 67 HTTPAuthHandler, 68 HTTPConnection, 69 HTTPSConnection, 70 ProxyHandler, 71 Request, 72 ) 73 74 75load_tests = load_tests_apply_scenarios 76 77 78def vary_by_http_client_implementation(): 79 """Test the libraries we can use, currently just urllib.""" 80 transport_scenarios = [ 81 ('urllib', dict(_transport=HttpTransport, 82 _server=http_server.HttpServer, 83 _url_protocol='http',)), 84 ] 85 return transport_scenarios 86 87 88def vary_by_http_protocol_version(): 89 """Test on http/1.0 and 1.1""" 90 return [ 91 ('HTTP/1.0', dict(_protocol_version='HTTP/1.0')), 92 ('HTTP/1.1', dict(_protocol_version='HTTP/1.1')), 93 ] 94 95 96def vary_by_http_auth_scheme(): 97 scenarios = [ 98 ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)), 99 ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)), 100 ('basicdigest', 101 dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)), 102 ] 103 # Add some attributes common to all scenarios 104 for scenario_id, scenario_dict in scenarios: 105 scenario_dict.update(_auth_header='Authorization', 106 _username_prompt_prefix='', 107 _password_prompt_prefix='') 108 return scenarios 109 110 111def vary_by_http_proxy_auth_scheme(): 112 scenarios = [ 113 ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)), 114 ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)), 115 ('proxy-basicdigest', 116 dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)), 117 ] 118 # Add some attributes common to all scenarios 119 for scenario_id, scenario_dict in scenarios: 120 scenario_dict.update(_auth_header='Proxy-Authorization', 121 _username_prompt_prefix='Proxy ', 122 _password_prompt_prefix='Proxy ') 123 return scenarios 124 125 126def vary_by_http_activity(): 127 activity_scenarios = [ 128 ('urllib,http', dict(_activity_server=ActivityHTTPServer, 129 _transport=HttpTransport,)), 130 ] 131 if features.HTTPSServerFeature.available(): 132 # FIXME: Until we have a better way to handle self-signed certificates 133 # (like allowing them in a test specific authentication.conf for 134 # example), we need some specialized urllib transport for tests. 135 # -- vila 2012-01-20 136 from . import ( 137 ssl_certs, 138 ) 139 140 class HTTPS_transport(HttpTransport): 141 142 def __init__(self, base, _from_transport=None): 143 super(HTTPS_transport, self).__init__( 144 base, _from_transport=_from_transport, 145 ca_certs=ssl_certs.build_path('ca.crt')) 146 147 activity_scenarios.append( 148 ('urllib,https', dict(_activity_server=ActivityHTTPSServer, 149 _transport=HTTPS_transport,)),) 150 return activity_scenarios 151 152 153class FakeManager(object): 154 155 def __init__(self): 156 self.credentials = [] 157 158 def add_password(self, realm, host, username, password): 159 self.credentials.append([realm, host, username, password]) 160 161 162class RecordingServer(object): 163 """A fake HTTP server. 164 165 It records the bytes sent to it, and replies with a 200. 166 """ 167 168 def __init__(self, expect_body_tail=None, scheme=''): 169 """Constructor. 170 171 :type expect_body_tail: str 172 :param expect_body_tail: a reply won't be sent until this string is 173 received. 174 """ 175 self._expect_body_tail = expect_body_tail 176 self.host = None 177 self.port = None 178 self.received_bytes = b'' 179 self.scheme = scheme 180 181 def get_url(self): 182 return '%s://%s:%s/' % (self.scheme, self.host, self.port) 183 184 def start_server(self): 185 self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 186 self._sock.bind(('127.0.0.1', 0)) 187 self.host, self.port = self._sock.getsockname() 188 self._ready = threading.Event() 189 self._thread = test_server.TestThread( 190 sync_event=self._ready, target=self._accept_read_and_reply) 191 self._thread.start() 192 if 'threads' in tests.selftest_debug_flags: 193 sys.stderr.write('Thread started: %s\n' % (self._thread.ident,)) 194 self._ready.wait() 195 196 def _accept_read_and_reply(self): 197 self._sock.listen(1) 198 self._ready.set() 199 conn, address = self._sock.accept() 200 if self._expect_body_tail is not None: 201 while not self.received_bytes.endswith(self._expect_body_tail): 202 self.received_bytes += conn.recv(4096) 203 conn.sendall(b'HTTP/1.1 200 OK\r\n') 204 try: 205 self._sock.close() 206 except socket.error: 207 # The client may have already closed the socket. 208 pass 209 210 def stop_server(self): 211 try: 212 # Issue a fake connection to wake up the server and allow it to 213 # finish quickly 214 fake_conn = osutils.connect_socket((self.host, self.port)) 215 fake_conn.close() 216 except socket.error: 217 # We might have already closed it. We don't care. 218 pass 219 self.host = None 220 self.port = None 221 self._thread.join() 222 if 'threads' in tests.selftest_debug_flags: 223 sys.stderr.write('Thread joined: %s\n' % (self._thread.ident,)) 224 225 226class TestAuthHeader(tests.TestCase): 227 228 def parse_header(self, header, auth_handler_class=None): 229 if auth_handler_class is None: 230 auth_handler_class = AbstractAuthHandler 231 self.auth_handler = auth_handler_class() 232 return self.auth_handler._parse_auth_header(header) 233 234 def test_empty_header(self): 235 scheme, remainder = self.parse_header('') 236 self.assertEqual('', scheme) 237 self.assertIs(None, remainder) 238 239 def test_negotiate_header(self): 240 scheme, remainder = self.parse_header('Negotiate') 241 self.assertEqual('negotiate', scheme) 242 self.assertIs(None, remainder) 243 244 def test_basic_header(self): 245 scheme, remainder = self.parse_header( 246 'Basic realm="Thou should not pass"') 247 self.assertEqual('basic', scheme) 248 self.assertEqual('realm="Thou should not pass"', remainder) 249 250 def test_build_basic_header_with_long_creds(self): 251 handler = BasicAuthHandler() 252 user = 'user' * 10 # length 40 253 password = 'password' * 5 # length 40 254 header = handler.build_auth_header( 255 dict(user=user, password=password), None) 256 # https://bugs.launchpad.net/bzr/+bug/1606203 was caused by incorrectly 257 # creating a header value with an embedded '\n' 258 self.assertFalse('\n' in header) 259 260 def test_basic_extract_realm(self): 261 scheme, remainder = self.parse_header( 262 'Basic realm="Thou should not pass"', 263 BasicAuthHandler) 264 match, realm = self.auth_handler.extract_realm(remainder) 265 self.assertTrue(match is not None) 266 self.assertEqual(u'Thou should not pass', realm) 267 268 def test_digest_header(self): 269 scheme, remainder = self.parse_header( 270 'Digest realm="Thou should not pass"') 271 self.assertEqual('digest', scheme) 272 self.assertEqual('realm="Thou should not pass"', remainder) 273 274 275class TestHTTPRangeParsing(tests.TestCase): 276 277 def setUp(self): 278 super(TestHTTPRangeParsing, self).setUp() 279 # We focus on range parsing here and ignore everything else 280 281 class RequestHandler(http_server.TestingHTTPRequestHandler): 282 def setup(self): pass 283 284 def handle(self): pass 285 286 def finish(self): pass 287 288 self.req_handler = RequestHandler(None, None, None) 289 290 def assertRanges(self, ranges, header, file_size): 291 self.assertEqual(ranges, 292 self.req_handler._parse_ranges(header, file_size)) 293 294 def test_simple_range(self): 295 self.assertRanges([(0, 2)], 'bytes=0-2', 12) 296 297 def test_tail(self): 298 self.assertRanges([(8, 11)], 'bytes=-4', 12) 299 300 def test_tail_bigger_than_file(self): 301 self.assertRanges([(0, 11)], 'bytes=-99', 12) 302 303 def test_range_without_end(self): 304 self.assertRanges([(4, 11)], 'bytes=4-', 12) 305 306 def test_invalid_ranges(self): 307 self.assertRanges(None, 'bytes=12-22', 12) 308 self.assertRanges(None, 'bytes=1-3,12-22', 12) 309 self.assertRanges(None, 'bytes=-', 12) 310 311 312class TestHTTPServer(tests.TestCase): 313 """Test the HTTP servers implementations.""" 314 315 def test_invalid_protocol(self): 316 class BogusRequestHandler(http_server.TestingHTTPRequestHandler): 317 318 protocol_version = 'HTTP/0.1' 319 320 self.assertRaises(UnknownProtocol, 321 http_server.HttpServer, BogusRequestHandler) 322 323 def test_force_invalid_protocol(self): 324 self.assertRaises(UnknownProtocol, 325 http_server.HttpServer, protocol_version='HTTP/0.1') 326 327 def test_server_start_and_stop(self): 328 server = http_server.HttpServer() 329 self.addCleanup(server.stop_server) 330 server.start_server() 331 self.assertTrue(server.server is not None) 332 self.assertTrue(server.server.serving is not None) 333 self.assertTrue(server.server.serving) 334 335 def test_create_http_server_one_zero(self): 336 class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler): 337 338 protocol_version = 'HTTP/1.0' 339 340 server = http_server.HttpServer(RequestHandlerOneZero) 341 self.start_server(server) 342 self.assertIsInstance(server.server, http_server.TestingHTTPServer) 343 344 def test_create_http_server_one_one(self): 345 class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler): 346 347 protocol_version = 'HTTP/1.1' 348 349 server = http_server.HttpServer(RequestHandlerOneOne) 350 self.start_server(server) 351 self.assertIsInstance(server.server, 352 http_server.TestingThreadingHTTPServer) 353 354 def test_create_http_server_force_one_one(self): 355 class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler): 356 357 protocol_version = 'HTTP/1.0' 358 359 server = http_server.HttpServer(RequestHandlerOneZero, 360 protocol_version='HTTP/1.1') 361 self.start_server(server) 362 self.assertIsInstance(server.server, 363 http_server.TestingThreadingHTTPServer) 364 365 def test_create_http_server_force_one_zero(self): 366 class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler): 367 368 protocol_version = 'HTTP/1.1' 369 370 server = http_server.HttpServer(RequestHandlerOneOne, 371 protocol_version='HTTP/1.0') 372 self.start_server(server) 373 self.assertIsInstance(server.server, 374 http_server.TestingHTTPServer) 375 376 377class TestHttpTransportUrls(tests.TestCase): 378 """Test the http urls.""" 379 380 scenarios = vary_by_http_client_implementation() 381 382 def test_abs_url(self): 383 """Construction of absolute http URLs""" 384 t = self._transport('http://example.com/bzr/bzr.dev/') 385 eq = self.assertEqualDiff 386 eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev') 387 eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar') 388 eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr') 389 eq(t.abspath('.bzr/1//2/./3'), 390 'http://example.com/bzr/bzr.dev/.bzr/1/2/3') 391 392 def test_invalid_http_urls(self): 393 """Trap invalid construction of urls""" 394 self._transport('http://example.com/bzr/bzr.dev/') 395 self.assertRaises(urlutils.InvalidURL, 396 self._transport, 397 'http://example.com:port/bzr/bzr.dev/') 398 399 def test_http_root_urls(self): 400 """Construction of URLs from server root""" 401 t = self._transport('http://example.com/') 402 eq = self.assertEqualDiff 403 eq(t.abspath('.bzr/tree-version'), 404 'http://example.com/.bzr/tree-version') 405 406 def test_http_impl_urls(self): 407 """There are servers which ask for particular clients to connect""" 408 server = self._server() 409 server.start_server() 410 try: 411 url = server.get_url() 412 self.assertTrue(url.startswith('%s://' % self._url_protocol)) 413 finally: 414 server.stop_server() 415 416 417class TestHTTPConnections(http_utils.TestCaseWithWebserver): 418 """Test the http connections.""" 419 420 scenarios = multiply_scenarios( 421 vary_by_http_client_implementation(), 422 vary_by_http_protocol_version(), 423 ) 424 425 def setUp(self): 426 super(TestHTTPConnections, self).setUp() 427 self.build_tree(['foo/', 'foo/bar'], line_endings='binary', 428 transport=self.get_transport()) 429 430 def test_http_has(self): 431 server = self.get_readonly_server() 432 t = self.get_readonly_transport() 433 self.assertEqual(t.has('foo/bar'), True) 434 self.assertEqual(len(server.logs), 1) 435 self.assertContainsRe(server.logs[0], 436 r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "Breezy/') 437 438 def test_http_has_not_found(self): 439 server = self.get_readonly_server() 440 t = self.get_readonly_transport() 441 self.assertEqual(t.has('not-found'), False) 442 self.assertContainsRe(server.logs[1], 443 r'"HEAD /not-found HTTP/1.." 404 - "-" "Breezy/') 444 445 def test_http_get(self): 446 server = self.get_readonly_server() 447 t = self.get_readonly_transport() 448 fp = t.get('foo/bar') 449 self.assertEqualDiff( 450 fp.read(), 451 b'contents of foo/bar\n') 452 self.assertEqual(len(server.logs), 1) 453 self.assertTrue(server.logs[0].find( 454 '"GET /foo/bar HTTP/1.1" 200 - "-" "Breezy/%s' 455 % breezy.__version__) > -1) 456 457 def test_has_on_bogus_host(self): 458 # Get a free address and don't 'accept' on it, so that we 459 # can be sure there is no http handler there, but set a 460 # reasonable timeout to not slow down tests too much. 461 default_timeout = socket.getdefaulttimeout() 462 try: 463 socket.setdefaulttimeout(2) 464 s = socket.socket() 465 s.bind(('localhost', 0)) 466 t = self._transport('http://%s:%s/' % s.getsockname()) 467 self.assertRaises(errors.ConnectionError, t.has, 'foo/bar') 468 finally: 469 socket.setdefaulttimeout(default_timeout) 470 471 472class TestHttpTransportRegistration(tests.TestCase): 473 """Test registrations of various http implementations""" 474 475 scenarios = vary_by_http_client_implementation() 476 477 def test_http_registered(self): 478 t = transport.get_transport_from_url( 479 '%s://foo.com/' % self._url_protocol) 480 self.assertIsInstance(t, transport.Transport) 481 self.assertIsInstance(t, self._transport) 482 483 484class TestPost(tests.TestCase): 485 486 scenarios = multiply_scenarios( 487 vary_by_http_client_implementation(), 488 vary_by_http_protocol_version(), 489 ) 490 491 def test_post_body_is_received(self): 492 server = RecordingServer(expect_body_tail=b'end-of-body', 493 scheme=self._url_protocol) 494 self.start_server(server) 495 url = server.get_url() 496 # FIXME: needs a cleanup -- vila 20100611 497 http_transport = transport.get_transport_from_url(url) 498 code, response = http_transport._post(b'abc def end-of-body') 499 self.assertTrue( 500 server.received_bytes.startswith(b'POST /.bzr/smart HTTP/1.')) 501 self.assertTrue( 502 b'content-length: 19\r' in server.received_bytes.lower()) 503 self.assertTrue(b'content-type: application/octet-stream\r' 504 in server.received_bytes.lower()) 505 # The transport should not be assuming that the server can accept 506 # chunked encoding the first time it connects, because HTTP/1.1, so we 507 # check for the literal string. 508 self.assertTrue( 509 server.received_bytes.endswith(b'\r\n\r\nabc def end-of-body')) 510 511 512class TestRangeHeader(tests.TestCase): 513 """Test range_header method""" 514 515 def check_header(self, value, ranges=[], tail=0): 516 offsets = [(start, end - start + 1) for start, end in ranges] 517 coalesce = transport.Transport._coalesce_offsets 518 coalesced = list(coalesce(offsets, limit=0, fudge_factor=0)) 519 range_header = HttpTransport._range_header 520 self.assertEqual(value, range_header(coalesced, tail)) 521 522 def test_range_header_single(self): 523 self.check_header('0-9', ranges=[(0, 9)]) 524 self.check_header('100-109', ranges=[(100, 109)]) 525 526 def test_range_header_tail(self): 527 self.check_header('-10', tail=10) 528 self.check_header('-50', tail=50) 529 530 def test_range_header_multi(self): 531 self.check_header('0-9,100-200,300-5000', 532 ranges=[(0, 9), (100, 200), (300, 5000)]) 533 534 def test_range_header_mixed(self): 535 self.check_header('0-9,300-5000,-50', 536 ranges=[(0, 9), (300, 5000)], 537 tail=50) 538 539 540class TestSpecificRequestHandler(http_utils.TestCaseWithWebserver): 541 """Tests a specific request handler. 542 543 Daughter classes are expected to override _req_handler_class 544 """ 545 546 scenarios = multiply_scenarios( 547 vary_by_http_client_implementation(), 548 vary_by_http_protocol_version(), 549 ) 550 551 # Provide a useful default 552 _req_handler_class = http_server.TestingHTTPRequestHandler 553 554 def create_transport_readonly_server(self): 555 server = http_server.HttpServer(self._req_handler_class, 556 protocol_version=self._protocol_version) 557 server._url_protocol = self._url_protocol 558 return server 559 560 561class WallRequestHandler(http_server.TestingHTTPRequestHandler): 562 """Whatever request comes in, close the connection""" 563 564 def _handle_one_request(self): 565 """Handle a single HTTP request, by abruptly closing the connection""" 566 self.close_connection = 1 567 568 569class TestWallServer(TestSpecificRequestHandler): 570 """Tests exceptions during the connection phase""" 571 572 _req_handler_class = WallRequestHandler 573 574 def test_http_has(self): 575 t = self.get_readonly_transport() 576 # Unfortunately httplib (see HTTPResponse._read_status 577 # for details) make no distinction between a closed 578 # socket and badly formatted status line, so we can't 579 # just test for ConnectionError, we have to test 580 # InvalidHttpResponse too. 581 self.assertRaises((errors.ConnectionError, 582 errors.InvalidHttpResponse), 583 t.has, 'foo/bar') 584 585 def test_http_get(self): 586 t = self.get_readonly_transport() 587 self.assertRaises((errors.ConnectionError, errors.ConnectionReset, 588 errors.InvalidHttpResponse), 589 t.get, 'foo/bar') 590 591 592class BadStatusRequestHandler(http_server.TestingHTTPRequestHandler): 593 """Whatever request comes in, returns a bad status""" 594 595 def parse_request(self): 596 """Fakes handling a single HTTP request, returns a bad status""" 597 ignored = http_server.TestingHTTPRequestHandler.parse_request(self) 598 self.send_response(0, "Bad status") 599 self.close_connection = 1 600 return False 601 602 603class TestBadStatusServer(TestSpecificRequestHandler): 604 """Tests bad status from server.""" 605 606 _req_handler_class = BadStatusRequestHandler 607 608 def setUp(self): 609 super(TestBadStatusServer, self).setUp() 610 # See https://bugs.launchpad.net/bzr/+bug/1451448 for details. 611 # TD;LR: Running both a TCP client and server in the same process and 612 # thread uncovers a race in python. The fix is to run the server in a 613 # different process. Trying to fix yet another race here is not worth 614 # the effort. -- vila 2015-09-06 615 if 'HTTP/1.0' in self.id(): 616 raise tests.TestSkipped( 617 'Client/Server in the same process and thread can hang') 618 619 def test_http_has(self): 620 t = self.get_readonly_transport() 621 self.assertRaises((errors.ConnectionError, errors.ConnectionReset, 622 errors.InvalidHttpResponse), 623 t.has, 'foo/bar') 624 625 def test_http_get(self): 626 t = self.get_readonly_transport() 627 self.assertRaises((errors.ConnectionError, errors.ConnectionReset, 628 errors.InvalidHttpResponse), 629 t.get, 'foo/bar') 630 631 632class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler): 633 """Whatever request comes in, returns an invalid status""" 634 635 def parse_request(self): 636 """Fakes handling a single HTTP request, returns a bad status""" 637 ignored = http_server.TestingHTTPRequestHandler.parse_request(self) 638 self.wfile.write(b"Invalid status line\r\n") 639 # If we don't close the connection pycurl will hang. Since this is a 640 # stress test we don't *have* to respect the protocol, but we don't 641 # have to sabotage it too much either. 642 self.close_connection = True 643 return False 644 645 646class TestInvalidStatusServer(TestBadStatusServer): 647 """Tests invalid status from server. 648 649 Both implementations raises the same error as for a bad status. 650 """ 651 652 _req_handler_class = InvalidStatusRequestHandler 653 654 655class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler): 656 """Whatever request comes in, returns a bad protocol version""" 657 658 def parse_request(self): 659 """Fakes handling a single HTTP request, returns a bad status""" 660 ignored = http_server.TestingHTTPRequestHandler.parse_request(self) 661 # Returns an invalid protocol version, but curl just 662 # ignores it and those cannot be tested. 663 self.wfile.write(b"%s %d %s\r\n" % ( 664 b'HTTP/0.0', 404, b'Look at my protocol version')) 665 return False 666 667 668class TestBadProtocolServer(TestSpecificRequestHandler): 669 """Tests bad protocol from server.""" 670 671 _req_handler_class = BadProtocolRequestHandler 672 673 def test_http_has(self): 674 t = self.get_readonly_transport() 675 self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar') 676 677 def test_http_get(self): 678 t = self.get_readonly_transport() 679 self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar') 680 681 682class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler): 683 """Whatever request comes in, returns a 403 code""" 684 685 def parse_request(self): 686 """Handle a single HTTP request, by replying we cannot handle it""" 687 ignored = http_server.TestingHTTPRequestHandler.parse_request(self) 688 self.send_error(403) 689 return False 690 691 692class TestForbiddenServer(TestSpecificRequestHandler): 693 """Tests forbidden server""" 694 695 _req_handler_class = ForbiddenRequestHandler 696 697 def test_http_has(self): 698 t = self.get_readonly_transport() 699 self.assertRaises(errors.TransportError, t.has, 'foo/bar') 700 701 def test_http_get(self): 702 t = self.get_readonly_transport() 703 self.assertRaises(errors.TransportError, t.get, 'foo/bar') 704 705 706class TestRecordingServer(tests.TestCase): 707 708 def test_create(self): 709 server = RecordingServer(expect_body_tail=None) 710 self.assertEqual(b'', server.received_bytes) 711 self.assertEqual(None, server.host) 712 self.assertEqual(None, server.port) 713 714 def test_setUp_and_stop(self): 715 server = RecordingServer(expect_body_tail=None) 716 server.start_server() 717 try: 718 self.assertNotEqual(None, server.host) 719 self.assertNotEqual(None, server.port) 720 finally: 721 server.stop_server() 722 self.assertEqual(None, server.host) 723 self.assertEqual(None, server.port) 724 725 def test_send_receive_bytes(self): 726 server = RecordingServer(expect_body_tail=b'c', scheme='http') 727 self.start_server(server) 728 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 729 sock.connect((server.host, server.port)) 730 sock.sendall(b'abc') 731 self.assertEqual(b'HTTP/1.1 200 OK\r\n', 732 osutils.recv_all(sock, 4096)) 733 self.assertEqual(b'abc', server.received_bytes) 734 735 736class TestRangeRequestServer(TestSpecificRequestHandler): 737 """Tests readv requests against server. 738 739 We test against default "normal" server. 740 """ 741 742 def setUp(self): 743 super(TestRangeRequestServer, self).setUp() 744 self.build_tree_contents([('a', b'0123456789')],) 745 746 def test_readv(self): 747 t = self.get_readonly_transport() 748 l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1)))) 749 self.assertEqual(l[0], (0, b'0')) 750 self.assertEqual(l[1], (1, b'1')) 751 self.assertEqual(l[2], (3, b'34')) 752 self.assertEqual(l[3], (9, b'9')) 753 754 def test_readv_out_of_order(self): 755 t = self.get_readonly_transport() 756 l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2)))) 757 self.assertEqual(l[0], (1, b'1')) 758 self.assertEqual(l[1], (9, b'9')) 759 self.assertEqual(l[2], (0, b'0')) 760 self.assertEqual(l[3], (3, b'34')) 761 762 def test_readv_invalid_ranges(self): 763 t = self.get_readonly_transport() 764 765 # This is intentionally reading off the end of the file 766 # since we are sure that it cannot get there 767 self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,), 768 t.readv, 'a', [(1, 1), (8, 10)]) 769 770 # This is trying to seek past the end of the file, it should 771 # also raise a special error 772 self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,), 773 t.readv, 'a', [(12, 2)]) 774 775 def test_readv_multiple_get_requests(self): 776 server = self.get_readonly_server() 777 t = self.get_readonly_transport() 778 # force transport to issue multiple requests 779 t._max_readv_combine = 1 780 t._max_get_ranges = 1 781 l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1)))) 782 self.assertEqual(l[0], (0, b'0')) 783 self.assertEqual(l[1], (1, b'1')) 784 self.assertEqual(l[2], (3, b'34')) 785 self.assertEqual(l[3], (9, b'9')) 786 # The server should have issued 4 requests 787 self.assertEqual(4, server.GET_request_nb) 788 789 def test_readv_get_max_size(self): 790 server = self.get_readonly_server() 791 t = self.get_readonly_transport() 792 # force transport to issue multiple requests by limiting the number of 793 # bytes by request. Note that this apply to coalesced offsets only, a 794 # single range will keep its size even if bigger than the limit. 795 t._get_max_size = 2 796 l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4)))) 797 self.assertEqual(l[0], (0, b'0')) 798 self.assertEqual(l[1], (1, b'1')) 799 self.assertEqual(l[2], (2, b'2345')) 800 self.assertEqual(l[3], (6, b'6789')) 801 # The server should have issued 3 requests 802 self.assertEqual(3, server.GET_request_nb) 803 804 def test_complete_readv_leave_pipe_clean(self): 805 server = self.get_readonly_server() 806 t = self.get_readonly_transport() 807 # force transport to issue multiple requests 808 t._get_max_size = 2 809 list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4)))) 810 # The server should have issued 3 requests 811 self.assertEqual(3, server.GET_request_nb) 812 self.assertEqual(b'0123456789', t.get_bytes('a')) 813 self.assertEqual(4, server.GET_request_nb) 814 815 def test_incomplete_readv_leave_pipe_clean(self): 816 server = self.get_readonly_server() 817 t = self.get_readonly_transport() 818 # force transport to issue multiple requests 819 t._get_max_size = 2 820 # Don't collapse readv results into a list so that we leave unread 821 # bytes on the socket 822 ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4)))) 823 self.assertEqual((0, b'0'), next(ireadv)) 824 # The server should have issued one request so far 825 self.assertEqual(1, server.GET_request_nb) 826 self.assertEqual(b'0123456789', t.get_bytes('a')) 827 # get_bytes issued an additional request, the readv pending ones are 828 # lost 829 self.assertEqual(2, server.GET_request_nb) 830 831 832class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler): 833 """Always reply to range request as if they were single. 834 835 Don't be explicit about it, just to annoy the clients. 836 """ 837 838 def get_multiple_ranges(self, file, file_size, ranges): 839 """Answer as if it was a single range request and ignores the rest""" 840 (start, end) = ranges[0] 841 return self.get_single_range(file, file_size, start, end) 842 843 844class TestSingleRangeRequestServer(TestRangeRequestServer): 845 """Test readv against a server which accept only single range requests""" 846 847 _req_handler_class = SingleRangeRequestHandler 848 849 850class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler): 851 """Only reply to simple range requests, errors out on multiple""" 852 853 def get_multiple_ranges(self, file, file_size, ranges): 854 """Refuses the multiple ranges request""" 855 if len(ranges) > 1: 856 file.close() 857 self.send_error(416, "Requested range not satisfiable") 858 return 859 (start, end) = ranges[0] 860 return self.get_single_range(file, file_size, start, end) 861 862 863class TestSingleOnlyRangeRequestServer(TestRangeRequestServer): 864 """Test readv against a server which only accept single range requests""" 865 866 _req_handler_class = SingleOnlyRangeRequestHandler 867 868 869class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler): 870 """Ignore range requests without notice""" 871 872 def do_GET(self): 873 # Update the statistics 874 self.server.test_case_server.GET_request_nb += 1 875 # Just bypass the range handling done by TestingHTTPRequestHandler 876 return SimpleHTTPRequestHandler.do_GET(self) 877 878 879class TestNoRangeRequestServer(TestRangeRequestServer): 880 """Test readv against a server which do not accept range requests""" 881 882 _req_handler_class = NoRangeRequestHandler 883 884 885class MultipleRangeWithoutContentLengthRequestHandler( 886 http_server.TestingHTTPRequestHandler): 887 """Reply to multiple range requests without content length header.""" 888 889 def get_multiple_ranges(self, file, file_size, ranges): 890 self.send_response(206) 891 self.send_header('Accept-Ranges', 'bytes') 892 # XXX: this is strange; the 'random' name below seems undefined and 893 # yet the tests pass -- mbp 2010-10-11 bug 658773 894 boundary = "%d" % random.randint(0, 0x7FFFFFFF) 895 self.send_header("Content-Type", 896 "multipart/byteranges; boundary=%s" % boundary) 897 self.end_headers() 898 for (start, end) in ranges: 899 self.wfile.write(b"--%s\r\n" % boundary.encode('ascii')) 900 self.send_header("Content-type", 'application/octet-stream') 901 self.send_header("Content-Range", "bytes %d-%d/%d" % (start, 902 end, 903 file_size)) 904 self.end_headers() 905 self.send_range_content(file, start, end - start + 1) 906 # Final boundary 907 self.wfile.write(b"--%s\r\n" % boundary) 908 909 910class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer): 911 912 _req_handler_class = MultipleRangeWithoutContentLengthRequestHandler 913 914 915class TruncatedMultipleRangeRequestHandler( 916 http_server.TestingHTTPRequestHandler): 917 """Reply to multiple range requests truncating the last ones. 918 919 This server generates responses whose Content-Length describes all the 920 ranges, but fail to include the last ones leading to client short reads. 921 This has been observed randomly with lighttpd (bug #179368). 922 """ 923 924 _truncated_ranges = 2 925 926 def get_multiple_ranges(self, file, file_size, ranges): 927 self.send_response(206) 928 self.send_header('Accept-Ranges', 'bytes') 929 boundary = 'tagada' 930 self.send_header('Content-Type', 931 'multipart/byteranges; boundary=%s' % boundary) 932 boundary_line = b'--%s\r\n' % boundary.encode('ascii') 933 # Calculate the Content-Length 934 content_length = 0 935 for (start, end) in ranges: 936 content_length += len(boundary_line) 937 content_length += self._header_line_length( 938 'Content-type', 'application/octet-stream') 939 content_length += self._header_line_length( 940 'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size)) 941 content_length += len('\r\n') # end headers 942 content_length += end - start # + 1 943 content_length += len(boundary_line) 944 self.send_header('Content-length', content_length) 945 self.end_headers() 946 947 # Send the multipart body 948 cur = 0 949 for (start, end) in ranges: 950 self.wfile.write(boundary_line) 951 self.send_header('Content-type', 'application/octet-stream') 952 self.send_header('Content-Range', 'bytes %d-%d/%d' 953 % (start, end, file_size)) 954 self.end_headers() 955 if cur + self._truncated_ranges >= len(ranges): 956 # Abruptly ends the response and close the connection 957 self.close_connection = 1 958 return 959 self.send_range_content(file, start, end - start + 1) 960 cur += 1 961 # Final boundary 962 self.wfile.write(boundary_line) 963 964 965class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler): 966 967 _req_handler_class = TruncatedMultipleRangeRequestHandler 968 969 def setUp(self): 970 super(TestTruncatedMultipleRangeServer, self).setUp() 971 self.build_tree_contents([('a', b'0123456789')],) 972 973 def test_readv_with_short_reads(self): 974 server = self.get_readonly_server() 975 t = self.get_readonly_transport() 976 # Force separate ranges for each offset 977 t._bytes_to_read_before_seek = 0 978 ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1)))) 979 self.assertEqual((0, b'0'), next(ireadv)) 980 self.assertEqual((2, b'2'), next(ireadv)) 981 # Only one request have been issued so far 982 self.assertEqual(1, server.GET_request_nb) 983 self.assertEqual((4, b'45'), next(ireadv)) 984 self.assertEqual((9, b'9'), next(ireadv)) 985 # We issue 3 requests: two multiple (4 ranges, then 2 ranges) then a 986 # single range. 987 self.assertEqual(3, server.GET_request_nb) 988 # Finally the client have tried a single range request and stays in 989 # that mode 990 self.assertEqual('single', t._range_hint) 991 992 993class TruncatedBeforeBoundaryRequestHandler( 994 http_server.TestingHTTPRequestHandler): 995 """Truncation before a boundary, like in bug 198646""" 996 997 _truncated_ranges = 1 998 999 def get_multiple_ranges(self, file, file_size, ranges): 1000 self.send_response(206) 1001 self.send_header('Accept-Ranges', 'bytes') 1002 boundary = 'tagada' 1003 self.send_header('Content-Type', 1004 'multipart/byteranges; boundary=%s' % boundary) 1005 boundary_line = b'--%s\r\n' % boundary.encode('ascii') 1006 # Calculate the Content-Length 1007 content_length = 0 1008 for (start, end) in ranges: 1009 content_length += len(boundary_line) 1010 content_length += self._header_line_length( 1011 'Content-type', 'application/octet-stream') 1012 content_length += self._header_line_length( 1013 'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size)) 1014 content_length += len('\r\n') # end headers 1015 content_length += end - start # + 1 1016 content_length += len(boundary_line) 1017 self.send_header('Content-length', content_length) 1018 self.end_headers() 1019 1020 # Send the multipart body 1021 cur = 0 1022 for (start, end) in ranges: 1023 if cur + self._truncated_ranges >= len(ranges): 1024 # Abruptly ends the response and close the connection 1025 self.close_connection = 1 1026 return 1027 self.wfile.write(boundary_line) 1028 self.send_header('Content-type', 'application/octet-stream') 1029 self.send_header('Content-Range', 'bytes %d-%d/%d' 1030 % (start, end, file_size)) 1031 self.end_headers() 1032 self.send_range_content(file, start, end - start + 1) 1033 cur += 1 1034 # Final boundary 1035 self.wfile.write(boundary_line) 1036 1037 1038class TestTruncatedBeforeBoundary(TestSpecificRequestHandler): 1039 """Tests the case of bug 198646, disconnecting before a boundary.""" 1040 1041 _req_handler_class = TruncatedBeforeBoundaryRequestHandler 1042 1043 def setUp(self): 1044 super(TestTruncatedBeforeBoundary, self).setUp() 1045 self.build_tree_contents([('a', b'0123456789')],) 1046 1047 def test_readv_with_short_reads(self): 1048 server = self.get_readonly_server() 1049 t = self.get_readonly_transport() 1050 # Force separate ranges for each offset 1051 t._bytes_to_read_before_seek = 0 1052 ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1)))) 1053 self.assertEqual((0, b'0'), next(ireadv)) 1054 self.assertEqual((2, b'2'), next(ireadv)) 1055 self.assertEqual((4, b'45'), next(ireadv)) 1056 self.assertEqual((9, b'9'), next(ireadv)) 1057 1058 1059class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler): 1060 """Errors out when range specifiers exceed the limit""" 1061 1062 def get_multiple_ranges(self, file, file_size, ranges): 1063 """Refuses the multiple ranges request""" 1064 tcs = self.server.test_case_server 1065 if tcs.range_limit is not None and len(ranges) > tcs.range_limit: 1066 file.close() 1067 # Emulate apache behavior 1068 self.send_error(400, "Bad Request") 1069 return 1070 return http_server.TestingHTTPRequestHandler.get_multiple_ranges( 1071 self, file, file_size, ranges) 1072 1073 1074class LimitedRangeHTTPServer(http_server.HttpServer): 1075 """An HttpServer erroring out on requests with too much range specifiers""" 1076 1077 def __init__(self, request_handler=LimitedRangeRequestHandler, 1078 protocol_version=None, 1079 range_limit=None): 1080 http_server.HttpServer.__init__(self, request_handler, 1081 protocol_version=protocol_version) 1082 self.range_limit = range_limit 1083 1084 1085class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver): 1086 """Tests readv requests against a server erroring out on too much ranges.""" 1087 1088 scenarios = multiply_scenarios( 1089 vary_by_http_client_implementation(), 1090 vary_by_http_protocol_version(), 1091 ) 1092 1093 # Requests with more range specifiers will error out 1094 range_limit = 3 1095 1096 def create_transport_readonly_server(self): 1097 return LimitedRangeHTTPServer(range_limit=self.range_limit, 1098 protocol_version=self._protocol_version) 1099 1100 def setUp(self): 1101 super(TestLimitedRangeRequestServer, self).setUp() 1102 # We need to manipulate ranges that correspond to real chunks in the 1103 # response, so we build a content appropriately. 1104 filler = b''.join([b'abcdefghij' for x in range(102)]) 1105 content = b''.join([b'%04d' % v + filler for v in range(16)]) 1106 self.build_tree_contents([('a', content)],) 1107 1108 def test_few_ranges(self): 1109 t = self.get_readonly_transport() 1110 l = list(t.readv('a', ((0, 4), (1024, 4), ))) 1111 self.assertEqual(l[0], (0, b'0000')) 1112 self.assertEqual(l[1], (1024, b'0001')) 1113 self.assertEqual(1, self.get_readonly_server().GET_request_nb) 1114 1115 def test_more_ranges(self): 1116 t = self.get_readonly_transport() 1117 l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4)))) 1118 self.assertEqual(l[0], (0, b'0000')) 1119 self.assertEqual(l[1], (1024, b'0001')) 1120 self.assertEqual(l[2], (4096, b'0004')) 1121 self.assertEqual(l[3], (8192, b'0008')) 1122 # The server will refuse to serve the first request (too much ranges), 1123 # a second request will succeed. 1124 self.assertEqual(2, self.get_readonly_server().GET_request_nb) 1125 1126 1127class TestHttpProxyWhiteBox(tests.TestCase): 1128 """Whitebox test proxy http authorization. 1129 1130 Only the urllib implementation is tested here. 1131 """ 1132 1133 def _proxied_request(self): 1134 handler = ProxyHandler() 1135 request = Request('GET', 'http://baz/buzzle') 1136 handler.set_proxy(request, 'http') 1137 return request 1138 1139 def assertEvaluateProxyBypass(self, expected, host, no_proxy): 1140 handler = ProxyHandler() 1141 self.assertEqual(expected, 1142 handler.evaluate_proxy_bypass(host, no_proxy)) 1143 1144 def test_empty_user(self): 1145 self.overrideEnv('http_proxy', 'http://bar.com') 1146 request = self._proxied_request() 1147 self.assertFalse('Proxy-authorization' in request.headers) 1148 1149 def test_user_with_at(self): 1150 self.overrideEnv('http_proxy', 1151 'http://username@domain:password@proxy_host:1234') 1152 request = self._proxied_request() 1153 self.assertFalse('Proxy-authorization' in request.headers) 1154 1155 def test_invalid_proxy(self): 1156 """A proxy env variable without scheme""" 1157 self.overrideEnv('http_proxy', 'host:1234') 1158 self.assertRaises(urlutils.InvalidURL, self._proxied_request) 1159 1160 def test_evaluate_proxy_bypass_true(self): 1161 """The host is not proxied""" 1162 self.assertEvaluateProxyBypass(True, 'example.com', 'example.com') 1163 self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com') 1164 1165 def test_evaluate_proxy_bypass_false(self): 1166 """The host is proxied""" 1167 self.assertEvaluateProxyBypass(False, 'bzr.example.com', None) 1168 1169 def test_evaluate_proxy_bypass_unknown(self): 1170 """The host is not explicitly proxied""" 1171 self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com') 1172 self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com') 1173 1174 def test_evaluate_proxy_bypass_empty_entries(self): 1175 """Ignore empty entries""" 1176 self.assertEvaluateProxyBypass(None, 'example.com', '') 1177 self.assertEvaluateProxyBypass(None, 'example.com', ',') 1178 self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar') 1179 1180 1181class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers): 1182 """Tests proxy server. 1183 1184 Be aware that we do not setup a real proxy here. Instead, we 1185 check that the *connection* goes through the proxy by serving 1186 different content (the faked proxy server append '-proxied' 1187 to the file names). 1188 """ 1189 1190 scenarios = multiply_scenarios( 1191 vary_by_http_client_implementation(), 1192 vary_by_http_protocol_version(), 1193 ) 1194 1195 # FIXME: We don't have an https server available, so we don't 1196 # test https connections. --vila toolongago 1197 1198 def setUp(self): 1199 super(TestProxyHttpServer, self).setUp() 1200 self.transport_secondary_server = http_utils.ProxyServer 1201 self.build_tree_contents([('foo', b'contents of foo\n'), 1202 ('foo-proxied', b'proxied contents of foo\n')]) 1203 # Let's setup some attributes for tests 1204 server = self.get_readonly_server() 1205 self.server_host_port = '%s:%d' % (server.host, server.port) 1206 self.no_proxy_host = self.server_host_port 1207 # The secondary server is the proxy 1208 self.proxy_url = self.get_secondary_url() 1209 1210 def assertProxied(self): 1211 t = self.get_readonly_transport() 1212 self.assertEqual(b'proxied contents of foo\n', t.get('foo').read()) 1213 1214 def assertNotProxied(self): 1215 t = self.get_readonly_transport() 1216 self.assertEqual(b'contents of foo\n', t.get('foo').read()) 1217 1218 def test_http_proxy(self): 1219 self.overrideEnv('http_proxy', self.proxy_url) 1220 self.assertProxied() 1221 1222 def test_HTTP_PROXY(self): 1223 self.overrideEnv('HTTP_PROXY', self.proxy_url) 1224 self.assertProxied() 1225 1226 def test_all_proxy(self): 1227 self.overrideEnv('all_proxy', self.proxy_url) 1228 self.assertProxied() 1229 1230 def test_ALL_PROXY(self): 1231 self.overrideEnv('ALL_PROXY', self.proxy_url) 1232 self.assertProxied() 1233 1234 def test_http_proxy_with_no_proxy(self): 1235 self.overrideEnv('no_proxy', self.no_proxy_host) 1236 self.overrideEnv('http_proxy', self.proxy_url) 1237 self.assertNotProxied() 1238 1239 def test_HTTP_PROXY_with_NO_PROXY(self): 1240 self.overrideEnv('NO_PROXY', self.no_proxy_host) 1241 self.overrideEnv('HTTP_PROXY', self.proxy_url) 1242 self.assertNotProxied() 1243 1244 def test_all_proxy_with_no_proxy(self): 1245 self.overrideEnv('no_proxy', self.no_proxy_host) 1246 self.overrideEnv('all_proxy', self.proxy_url) 1247 self.assertNotProxied() 1248 1249 def test_ALL_PROXY_with_NO_PROXY(self): 1250 self.overrideEnv('NO_PROXY', self.no_proxy_host) 1251 self.overrideEnv('ALL_PROXY', self.proxy_url) 1252 self.assertNotProxied() 1253 1254 def test_http_proxy_without_scheme(self): 1255 self.overrideEnv('http_proxy', self.server_host_port) 1256 self.assertRaises(urlutils.InvalidURL, self.assertProxied) 1257 1258 1259class TestRanges(http_utils.TestCaseWithWebserver): 1260 """Test the Range header in GET methods.""" 1261 1262 scenarios = multiply_scenarios( 1263 vary_by_http_client_implementation(), 1264 vary_by_http_protocol_version(), 1265 ) 1266 1267 def setUp(self): 1268 super(TestRanges, self).setUp() 1269 self.build_tree_contents([('a', b'0123456789')],) 1270 1271 def create_transport_readonly_server(self): 1272 return http_server.HttpServer(protocol_version=self._protocol_version) 1273 1274 def _file_contents(self, relpath, ranges): 1275 t = self.get_readonly_transport() 1276 offsets = [(start, end - start + 1) for start, end in ranges] 1277 coalesce = t._coalesce_offsets 1278 coalesced = list(coalesce(offsets, limit=0, fudge_factor=0)) 1279 code, data = t._get(relpath, coalesced) 1280 self.assertTrue(code in (200, 206), '_get returns: %d' % code) 1281 for start, end in ranges: 1282 data.seek(start) 1283 yield data.read(end - start + 1) 1284 1285 def _file_tail(self, relpath, tail_amount): 1286 t = self.get_readonly_transport() 1287 code, data = t._get(relpath, [], tail_amount) 1288 self.assertTrue(code in (200, 206), '_get returns: %d' % code) 1289 data.seek(-tail_amount, 2) 1290 return data.read(tail_amount) 1291 1292 def test_range_header(self): 1293 # Valid ranges 1294 self.assertEqual( 1295 [b'0', b'234'], list(self._file_contents('a', [(0, 0), (2, 4)]))) 1296 1297 def test_range_header_tail(self): 1298 self.assertEqual(b'789', self._file_tail('a', 3)) 1299 1300 def test_syntactically_invalid_range_header(self): 1301 self.assertListRaises(errors.InvalidHttpRange, 1302 self._file_contents, 'a', [(4, 3)]) 1303 1304 def test_semantically_invalid_range_header(self): 1305 self.assertListRaises(errors.InvalidHttpRange, 1306 self._file_contents, 'a', [(42, 128)]) 1307 1308 1309class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver): 1310 """Test redirection between http servers.""" 1311 1312 scenarios = multiply_scenarios( 1313 vary_by_http_client_implementation(), 1314 vary_by_http_protocol_version(), 1315 ) 1316 1317 def setUp(self): 1318 super(TestHTTPRedirections, self).setUp() 1319 self.build_tree_contents([('a', b'0123456789'), 1320 ('bundle', 1321 b'# Bazaar revision bundle v0.9\n#\n') 1322 ],) 1323 1324 def test_redirected(self): 1325 self.assertRaises(errors.RedirectRequested, 1326 self.get_old_transport().get, 'a') 1327 self.assertEqual( 1328 b'0123456789', 1329 self.get_new_transport().get('a').read()) 1330 1331 1332class RedirectedRequest(Request): 1333 """Request following redirections. """ 1334 1335 init_orig = Request.__init__ 1336 1337 def __init__(self, method, url, *args, **kwargs): 1338 """Constructor. 1339 1340 """ 1341 # Since the tests using this class will replace 1342 # Request, we can't just call the base class __init__ 1343 # or we'll loop. 1344 RedirectedRequest.init_orig(self, method, url, *args, **kwargs) 1345 self.follow_redirections = True 1346 1347 1348def install_redirected_request(test): 1349 test.overrideAttr(urllib, 'Request', RedirectedRequest) 1350 1351 1352def cleanup_http_redirection_connections(test): 1353 # Some sockets are opened but never seen by _urllib, so we trap them at 1354 # the http level to be able to clean them up. 1355 def socket_disconnect(sock): 1356 try: 1357 sock.shutdown(socket.SHUT_RDWR) 1358 sock.close() 1359 except socket.error: 1360 pass 1361 1362 def connect(connection): 1363 test.http_connect_orig(connection) 1364 test.addCleanup(socket_disconnect, connection.sock) 1365 test.http_connect_orig = test.overrideAttr( 1366 HTTPConnection, 'connect', connect) 1367 1368 def connect(connection): 1369 test.https_connect_orig(connection) 1370 test.addCleanup(socket_disconnect, connection.sock) 1371 test.https_connect_orig = test.overrideAttr( 1372 HTTPSConnection, 'connect', connect) 1373 1374 1375class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver): 1376 """Test redirections. 1377 1378 http implementations do not redirect silently anymore (they 1379 do not redirect at all in fact). The mechanism is still in 1380 place at the Request level and these tests 1381 exercise it. 1382 """ 1383 1384 scenarios = multiply_scenarios( 1385 vary_by_http_client_implementation(), 1386 vary_by_http_protocol_version(), 1387 ) 1388 1389 def setUp(self): 1390 super(TestHTTPSilentRedirections, self).setUp() 1391 install_redirected_request(self) 1392 cleanup_http_redirection_connections(self) 1393 self.build_tree_contents([('a', b'a'), 1394 ('1/',), 1395 ('1/a', b'redirected once'), 1396 ('2/',), 1397 ('2/a', b'redirected twice'), 1398 ('3/',), 1399 ('3/a', b'redirected thrice'), 1400 ('4/',), 1401 ('4/a', b'redirected 4 times'), 1402 ('5/',), 1403 ('5/a', b'redirected 5 times'), 1404 ],) 1405 1406 def test_one_redirection(self): 1407 t = self.get_old_transport() 1408 new_prefix = 'http://%s:%s' % (self.new_server.host, 1409 self.new_server.port) 1410 self.old_server.redirections = \ 1411 [('(.*)', r'%s/1\1' % (new_prefix), 301), ] 1412 self.assertEqual( 1413 b'redirected once', 1414 t.request('GET', t._remote_path('a'), retries=1).read()) 1415 1416 def test_five_redirections(self): 1417 t = self.get_old_transport() 1418 old_prefix = 'http://%s:%s' % (self.old_server.host, 1419 self.old_server.port) 1420 new_prefix = 'http://%s:%s' % (self.new_server.host, 1421 self.new_server.port) 1422 self.old_server.redirections = [ 1423 ('/1(.*)', r'%s/2\1' % (old_prefix), 302), 1424 ('/2(.*)', r'%s/3\1' % (old_prefix), 303), 1425 ('/3(.*)', r'%s/4\1' % (old_prefix), 307), 1426 ('/4(.*)', r'%s/5\1' % (new_prefix), 301), 1427 ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301), 1428 ] 1429 self.assertEqual( 1430 b'redirected 5 times', 1431 t.request('GET', t._remote_path('a'), retries=6).read()) 1432 1433 1434class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver): 1435 """Test transport.do_catching_redirections.""" 1436 1437 scenarios = multiply_scenarios( 1438 vary_by_http_client_implementation(), 1439 vary_by_http_protocol_version(), 1440 ) 1441 1442 def setUp(self): 1443 super(TestDoCatchRedirections, self).setUp() 1444 self.build_tree_contents([('a', b'0123456789'), ],) 1445 cleanup_http_redirection_connections(self) 1446 1447 self.old_transport = self.get_old_transport() 1448 1449 def get_a(self, t): 1450 return t.get('a') 1451 1452 def test_no_redirection(self): 1453 t = self.get_new_transport() 1454 1455 # We use None for redirected so that we fail if redirected 1456 self.assertEqual(b'0123456789', 1457 transport.do_catching_redirections( 1458 self.get_a, t, None).read()) 1459 1460 def test_one_redirection(self): 1461 self.redirections = 0 1462 1463 def redirected(t, exception, redirection_notice): 1464 self.redirections += 1 1465 redirected_t = t._redirected_to(exception.source, exception.target) 1466 return redirected_t 1467 1468 self.assertEqual(b'0123456789', 1469 transport.do_catching_redirections( 1470 self.get_a, self.old_transport, redirected).read()) 1471 self.assertEqual(1, self.redirections) 1472 1473 def test_redirection_loop(self): 1474 1475 def redirected(transport, exception, redirection_notice): 1476 # By using the redirected url as a base dir for the 1477 # *old* transport, we create a loop: a => a/a => 1478 # a/a/a 1479 return self.old_transport.clone(exception.target) 1480 1481 self.assertRaises(errors.TooManyRedirections, 1482 transport.do_catching_redirections, 1483 self.get_a, self.old_transport, redirected) 1484 1485 1486def _setup_authentication_config(**kwargs): 1487 conf = config.AuthenticationConfig() 1488 conf._get_config().update({'httptest': kwargs}) 1489 conf._save() 1490 1491 1492class TestUrllib2AuthHandler(tests.TestCaseWithTransport): 1493 """Unit tests for glue by which urllib2 asks us for authentication""" 1494 1495 def test_get_user_password_without_port(self): 1496 """We cope if urllib2 doesn't tell us the port. 1497 1498 See https://bugs.launchpad.net/bzr/+bug/654684 1499 """ 1500 user = 'joe' 1501 password = 'foo' 1502 _setup_authentication_config(scheme='http', host='localhost', 1503 user=user, password=password) 1504 handler = HTTPAuthHandler() 1505 got_pass = handler.get_user_password(dict( 1506 user='joe', 1507 protocol='http', 1508 host='localhost', 1509 path='/', 1510 realm=u'Realm', 1511 )) 1512 self.assertEqual((user, password), got_pass) 1513 1514 1515class TestAuth(http_utils.TestCaseWithWebserver): 1516 """Test authentication scheme""" 1517 1518 scenarios = multiply_scenarios( 1519 vary_by_http_client_implementation(), 1520 vary_by_http_protocol_version(), 1521 vary_by_http_auth_scheme(), 1522 ) 1523 1524 def setUp(self): 1525 super(TestAuth, self).setUp() 1526 self.server = self.get_readonly_server() 1527 self.build_tree_contents([('a', b'contents of a\n'), 1528 ('b', b'contents of b\n'), ]) 1529 1530 def create_transport_readonly_server(self): 1531 server = self._auth_server(protocol_version=self._protocol_version) 1532 server._url_protocol = self._url_protocol 1533 return server 1534 1535 def get_user_url(self, user, password): 1536 """Build an url embedding user and password""" 1537 url = '%s://' % self.server._url_protocol 1538 if user is not None: 1539 url += user 1540 if password is not None: 1541 url += ':' + password 1542 url += '@' 1543 url += '%s:%s/' % (self.server.host, self.server.port) 1544 return url 1545 1546 def get_user_transport(self, user, password): 1547 t = transport.get_transport_from_url( 1548 self.get_user_url(user, password)) 1549 return t 1550 1551 def test_no_user(self): 1552 self.server.add_user('joe', 'foo') 1553 t = self.get_user_transport(None, None) 1554 self.assertRaises(errors.InvalidHttpResponse, t.get, 'a') 1555 # Only one 'Authentication Required' error should occur 1556 self.assertEqual(1, self.server.auth_required_errors) 1557 1558 def test_empty_pass(self): 1559 self.server.add_user('joe', '') 1560 t = self.get_user_transport('joe', '') 1561 self.assertEqual(b'contents of a\n', t.get('a').read()) 1562 # Only one 'Authentication Required' error should occur 1563 self.assertEqual(1, self.server.auth_required_errors) 1564 1565 def test_user_pass(self): 1566 self.server.add_user('joe', 'foo') 1567 t = self.get_user_transport('joe', 'foo') 1568 self.assertEqual(b'contents of a\n', t.get('a').read()) 1569 # Only one 'Authentication Required' error should occur 1570 self.assertEqual(1, self.server.auth_required_errors) 1571 1572 def test_unknown_user(self): 1573 self.server.add_user('joe', 'foo') 1574 t = self.get_user_transport('bill', 'foo') 1575 self.assertRaises(errors.InvalidHttpResponse, t.get, 'a') 1576 # Two 'Authentication Required' errors should occur (the 1577 # initial 'who are you' and 'I don't know you, who are 1578 # you'). 1579 self.assertEqual(2, self.server.auth_required_errors) 1580 1581 def test_wrong_pass(self): 1582 self.server.add_user('joe', 'foo') 1583 t = self.get_user_transport('joe', 'bar') 1584 self.assertRaises(errors.InvalidHttpResponse, t.get, 'a') 1585 # Two 'Authentication Required' errors should occur (the 1586 # initial 'who are you' and 'this is not you, who are you') 1587 self.assertEqual(2, self.server.auth_required_errors) 1588 1589 def test_prompt_for_username(self): 1590 self.server.add_user('joe', 'foo') 1591 t = self.get_user_transport(None, None) 1592 ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n') 1593 stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr 1594 self.assertEqual(b'contents of a\n', t.get('a').read()) 1595 # stdin should be empty 1596 self.assertEqual('', ui.ui_factory.stdin.readline()) 1597 stderr.seek(0) 1598 expected_prompt = self._expected_username_prompt(t._unqualified_scheme) 1599 self.assertEqual(expected_prompt, stderr.read(len(expected_prompt))) 1600 self.assertEqual('', stdout.getvalue()) 1601 self._check_password_prompt(t._unqualified_scheme, 'joe', 1602 stderr.readline()) 1603 1604 def test_prompt_for_password(self): 1605 self.server.add_user('joe', 'foo') 1606 t = self.get_user_transport('joe', None) 1607 ui.ui_factory = tests.TestUIFactory(stdin='foo\n') 1608 stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr 1609 self.assertEqual(b'contents of a\n', t.get('a').read()) 1610 # stdin should be empty 1611 self.assertEqual('', ui.ui_factory.stdin.readline()) 1612 self._check_password_prompt(t._unqualified_scheme, 'joe', 1613 stderr.getvalue()) 1614 self.assertEqual('', stdout.getvalue()) 1615 # And we shouldn't prompt again for a different request 1616 # against the same transport. 1617 self.assertEqual(b'contents of b\n', t.get('b').read()) 1618 t2 = t.clone() 1619 # And neither against a clone 1620 self.assertEqual(b'contents of b\n', t2.get('b').read()) 1621 # Only one 'Authentication Required' error should occur 1622 self.assertEqual(1, self.server.auth_required_errors) 1623 1624 def _check_password_prompt(self, scheme, user, actual_prompt): 1625 expected_prompt = (self._password_prompt_prefix 1626 + ("%s %s@%s:%d, Realm: '%s' password: " 1627 % (scheme.upper(), 1628 user, self.server.host, self.server.port, 1629 self.server.auth_realm))) 1630 self.assertEqual(expected_prompt, actual_prompt) 1631 1632 def _expected_username_prompt(self, scheme): 1633 return (self._username_prompt_prefix 1634 + "%s %s:%d, Realm: '%s' username: " % (scheme.upper(), 1635 self.server.host, self.server.port, 1636 self.server.auth_realm)) 1637 1638 def test_no_prompt_for_password_when_using_auth_config(self): 1639 user = ' joe' 1640 password = 'foo' 1641 stdin_content = 'bar\n' # Not the right password 1642 self.server.add_user(user, password) 1643 t = self.get_user_transport(user, None) 1644 ui.ui_factory = tests.TestUIFactory(stdin=stdin_content) 1645 # Create a minimal config file with the right password 1646 _setup_authentication_config(scheme='http', port=self.server.port, 1647 user=user, password=password) 1648 # Issue a request to the server to connect 1649 with t.get('a') as f: 1650 self.assertEqual(b'contents of a\n', f.read()) 1651 # stdin should have been left untouched 1652 self.assertEqual(stdin_content, ui.ui_factory.stdin.readline()) 1653 # Only one 'Authentication Required' error should occur 1654 self.assertEqual(1, self.server.auth_required_errors) 1655 1656 def test_changing_nonce(self): 1657 if self._auth_server not in (http_utils.HTTPDigestAuthServer, 1658 http_utils.ProxyDigestAuthServer): 1659 raise tests.TestNotApplicable('HTTP/proxy auth digest only test') 1660 self.server.add_user('joe', 'foo') 1661 t = self.get_user_transport('joe', 'foo') 1662 with t.get('a') as f: 1663 self.assertEqual(b'contents of a\n', f.read()) 1664 with t.get('b') as f: 1665 self.assertEqual(b'contents of b\n', f.read()) 1666 # Only one 'Authentication Required' error should have 1667 # occured so far 1668 self.assertEqual(1, self.server.auth_required_errors) 1669 # The server invalidates the current nonce 1670 self.server.auth_nonce = self.server.auth_nonce + '. No, now!' 1671 self.assertEqual(b'contents of a\n', t.get('a').read()) 1672 # Two 'Authentication Required' errors should occur (the 1673 # initial 'who are you' and a second 'who are you' with the new nonce) 1674 self.assertEqual(2, self.server.auth_required_errors) 1675 1676 def test_user_from_auth_conf(self): 1677 user = 'joe' 1678 password = 'foo' 1679 self.server.add_user(user, password) 1680 _setup_authentication_config(scheme='http', port=self.server.port, 1681 user=user, password=password) 1682 t = self.get_user_transport(None, None) 1683 # Issue a request to the server to connect 1684 with t.get('a') as f: 1685 self.assertEqual(b'contents of a\n', f.read()) 1686 # Only one 'Authentication Required' error should occur 1687 self.assertEqual(1, self.server.auth_required_errors) 1688 1689 def test_no_credential_leaks_in_log(self): 1690 self.overrideAttr(debug, 'debug_flags', {'http'}) 1691 user = 'joe' 1692 password = 'very-sensitive-password' 1693 self.server.add_user(user, password) 1694 t = self.get_user_transport(user, password) 1695 # Capture the debug calls to mutter 1696 self.mutters = [] 1697 1698 def mutter(*args): 1699 lines = args[0] % args[1:] 1700 # Some calls output multiple lines, just split them now since we 1701 # care about a single one later. 1702 self.mutters.extend(lines.splitlines()) 1703 self.overrideAttr(trace, 'mutter', mutter) 1704 # Issue a request to the server to connect 1705 self.assertEqual(True, t.has('a')) 1706 # Only one 'Authentication Required' error should occur 1707 self.assertEqual(1, self.server.auth_required_errors) 1708 # Since the authentification succeeded, there should be a corresponding 1709 # debug line 1710 sent_auth_headers = [line for line in self.mutters 1711 if line.startswith('> %s' % (self._auth_header,))] 1712 self.assertLength(1, sent_auth_headers) 1713 self.assertStartsWith(sent_auth_headers[0], 1714 '> %s: <masked>' % (self._auth_header,)) 1715 1716 1717class TestProxyAuth(TestAuth): 1718 """Test proxy authentication schemes. 1719 1720 This inherits from TestAuth to tweak the setUp and filter some failing 1721 tests. 1722 """ 1723 1724 scenarios = multiply_scenarios( 1725 vary_by_http_client_implementation(), 1726 vary_by_http_protocol_version(), 1727 vary_by_http_proxy_auth_scheme(), 1728 ) 1729 1730 def setUp(self): 1731 super(TestProxyAuth, self).setUp() 1732 # Override the contents to avoid false positives 1733 self.build_tree_contents([('a', b'not proxied contents of a\n'), 1734 ('b', b'not proxied contents of b\n'), 1735 ('a-proxied', b'contents of a\n'), 1736 ('b-proxied', b'contents of b\n'), 1737 ]) 1738 1739 def get_user_transport(self, user, password): 1740 proxy_url = self.get_user_url(user, password) 1741 self.overrideEnv('all_proxy', proxy_url) 1742 return TestAuth.get_user_transport(self, user, password) 1743 1744 1745class NonClosingBytesIO(io.BytesIO): 1746 1747 def close(self): 1748 """Ignore and leave file open.""" 1749 1750 1751class SampleSocket(object): 1752 """A socket-like object for use in testing the HTTP request handler.""" 1753 1754 def __init__(self, socket_read_content): 1755 """Constructs a sample socket. 1756 1757 :param socket_read_content: a byte sequence 1758 """ 1759 self.readfile = io.BytesIO(socket_read_content) 1760 self.writefile = NonClosingBytesIO() 1761 1762 def close(self): 1763 """Ignore and leave files alone.""" 1764 1765 def sendall(self, bytes): 1766 self.writefile.write(bytes) 1767 1768 def makefile(self, mode='r', bufsize=None): 1769 if 'r' in mode: 1770 return self.readfile 1771 else: 1772 return self.writefile 1773 1774 1775class SmartHTTPTunnellingTest(tests.TestCaseWithTransport): 1776 1777 scenarios = multiply_scenarios( 1778 vary_by_http_client_implementation(), 1779 vary_by_http_protocol_version(), 1780 ) 1781 1782 def setUp(self): 1783 super(SmartHTTPTunnellingTest, self).setUp() 1784 # We use the VFS layer as part of HTTP tunnelling tests. 1785 self.overrideEnv('BRZ_NO_SMART_VFS', None) 1786 self.transport_readonly_server = http_utils.HTTPServerWithSmarts 1787 self.http_server = self.get_readonly_server() 1788 1789 def create_transport_readonly_server(self): 1790 server = http_utils.HTTPServerWithSmarts( 1791 protocol_version=self._protocol_version) 1792 server._url_protocol = self._url_protocol 1793 return server 1794 1795 def test_open_controldir(self): 1796 branch = self.make_branch('relpath') 1797 url = self.http_server.get_url() + 'relpath' 1798 bd = controldir.ControlDir.open(url) 1799 self.addCleanup(bd.transport.disconnect) 1800 self.assertIsInstance(bd, _mod_remote.RemoteBzrDir) 1801 1802 def test_bulk_data(self): 1803 # We should be able to send and receive bulk data in a single message. 1804 # The 'readv' command in the smart protocol both sends and receives 1805 # bulk data, so we use that. 1806 self.build_tree(['data-file']) 1807 http_transport = transport.get_transport_from_url( 1808 self.http_server.get_url()) 1809 medium = http_transport.get_smart_medium() 1810 # Since we provide the medium, the url below will be mostly ignored 1811 # during the test, as long as the path is '/'. 1812 remote_transport = remote.RemoteTransport('bzr://fake_host/', 1813 medium=medium) 1814 self.assertEqual( 1815 [(0, b"c")], list(remote_transport.readv("data-file", [(0, 1)]))) 1816 1817 def test_http_send_smart_request(self): 1818 1819 post_body = b'hello\n' 1820 expected_reply_body = b'ok\x012\n' 1821 1822 http_transport = transport.get_transport_from_url( 1823 self.http_server.get_url()) 1824 medium = http_transport.get_smart_medium() 1825 response = medium.send_http_smart_request(post_body) 1826 reply_body = response.read() 1827 self.assertEqual(expected_reply_body, reply_body) 1828 1829 def test_smart_http_server_post_request_handler(self): 1830 httpd = self.http_server.server 1831 1832 socket = SampleSocket( 1833 b'POST /.bzr/smart %s \r\n' % self._protocol_version.encode('ascii') + 1834 # HTTP/1.1 posts must have a Content-Length (but it doesn't hurt 1835 # for 1.0) 1836 b'Content-Length: 6\r\n' 1837 b'\r\n' 1838 b'hello\n') 1839 # Beware: the ('localhost', 80) below is the 1840 # client_address parameter, but we don't have one because 1841 # we have defined a socket which is not bound to an 1842 # address. The test framework never uses this client 1843 # address, so far... 1844 request_handler = http_utils.SmartRequestHandler(socket, 1845 ('localhost', 80), 1846 httpd) 1847 response = socket.writefile.getvalue() 1848 self.assertStartsWith( 1849 response, 1850 b'%s 200 ' % self._protocol_version.encode('ascii')) 1851 # This includes the end of the HTTP headers, and all the body. 1852 expected_end_of_response = b'\r\n\r\nok\x012\n' 1853 self.assertEndsWith(response, expected_end_of_response) 1854 1855 1856class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler): 1857 """No smart server here request handler.""" 1858 1859 def do_POST(self): 1860 self.send_error(403, "Forbidden") 1861 1862 1863class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler): 1864 """Test smart client behaviour against an http server without smarts.""" 1865 1866 _req_handler_class = ForbiddenRequestHandler 1867 1868 def test_probe_smart_server(self): 1869 """Test error handling against server refusing smart requests.""" 1870 t = self.get_readonly_transport() 1871 # No need to build a valid smart request here, the server will not even 1872 # try to interpret it. 1873 self.assertRaises(errors.SmartProtocolError, 1874 t.get_smart_medium().send_http_smart_request, 1875 b'whatever') 1876 1877 1878class Test_redirected_to(tests.TestCase): 1879 1880 scenarios = vary_by_http_client_implementation() 1881 1882 def test_redirected_to_subdir(self): 1883 t = self._transport('http://www.example.com/foo') 1884 r = t._redirected_to('http://www.example.com/foo', 1885 'http://www.example.com/foo/subdir') 1886 self.assertIsInstance(r, type(t)) 1887 # Both transports share the some connection 1888 self.assertEqual(t._get_connection(), r._get_connection()) 1889 self.assertEqual('http://www.example.com/foo/subdir/', r.base) 1890 1891 def test_redirected_to_self_with_slash(self): 1892 t = self._transport('http://www.example.com/foo') 1893 r = t._redirected_to('http://www.example.com/foo', 1894 'http://www.example.com/foo/') 1895 self.assertIsInstance(r, type(t)) 1896 # Both transports share the some connection (one can argue that we 1897 # should return the exact same transport here, but that seems 1898 # overkill). 1899 self.assertEqual(t._get_connection(), r._get_connection()) 1900 1901 def test_redirected_to_host(self): 1902 t = self._transport('http://www.example.com/foo') 1903 r = t._redirected_to('http://www.example.com/foo', 1904 'http://foo.example.com/foo/subdir') 1905 self.assertIsInstance(r, type(t)) 1906 self.assertEqual('http://foo.example.com/foo/subdir/', 1907 r.external_url()) 1908 1909 def test_redirected_to_same_host_sibling_protocol(self): 1910 t = self._transport('http://www.example.com/foo') 1911 r = t._redirected_to('http://www.example.com/foo', 1912 'https://www.example.com/foo') 1913 self.assertIsInstance(r, type(t)) 1914 self.assertEqual('https://www.example.com/foo/', 1915 r.external_url()) 1916 1917 def test_redirected_to_same_host_different_protocol(self): 1918 t = self._transport('http://www.example.com/foo') 1919 r = t._redirected_to('http://www.example.com/foo', 1920 'bzr://www.example.com/foo') 1921 self.assertNotEqual(type(r), type(t)) 1922 self.assertEqual('bzr://www.example.com/foo/', r.external_url()) 1923 1924 def test_redirected_to_same_host_specific_implementation(self): 1925 t = self._transport('http://www.example.com/foo') 1926 r = t._redirected_to('http://www.example.com/foo', 1927 'https+urllib://www.example.com/foo') 1928 self.assertEqual('https://www.example.com/foo/', r.external_url()) 1929 1930 def test_redirected_to_different_host_same_user(self): 1931 t = self._transport('http://joe@www.example.com/foo') 1932 r = t._redirected_to('http://www.example.com/foo', 1933 'https://foo.example.com/foo') 1934 self.assertIsInstance(r, type(t)) 1935 self.assertEqual(t._parsed_url.user, r._parsed_url.user) 1936 self.assertEqual('https://joe@foo.example.com/foo/', r.external_url()) 1937 1938 1939class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler): 1940 """Request handler for a unique and pre-defined request. 1941 1942 The only thing we care about here is how many bytes travel on the wire. But 1943 since we want to measure it for a real http client, we have to send it 1944 correct responses. 1945 1946 We expect to receive a *single* request nothing more (and we won't even 1947 check what request it is, we just measure the bytes read until an empty 1948 line. 1949 """ 1950 1951 def _handle_one_request(self): 1952 tcs = self.server.test_case_server 1953 requestline = self.rfile.readline() 1954 headers = parse_headers(self.rfile) 1955 bytes_read = len(headers.as_bytes()) 1956 bytes_read += headers.as_bytes().count(b'\n') 1957 bytes_read += len(requestline) 1958 if requestline.startswith(b'POST'): 1959 # The body should be a single line (or we don't know where it ends 1960 # and we don't want to issue a blocking read) 1961 body = self.rfile.readline() 1962 bytes_read += len(body) 1963 tcs.bytes_read = bytes_read 1964 1965 # We set the bytes written *before* issuing the write, the client is 1966 # supposed to consume every produced byte *before* checking that value. 1967 1968 # Doing the oppposite may lead to test failure: we may be interrupted 1969 # after the write but before updating the value. The client can then 1970 # continue and read the value *before* we can update it. And yes, 1971 # this has been observed -- vila 20090129 1972 tcs.bytes_written = len(tcs.canned_response) 1973 self.wfile.write(tcs.canned_response) 1974 1975 1976class ActivityServerMixin(object): 1977 1978 def __init__(self, protocol_version): 1979 super(ActivityServerMixin, self).__init__( 1980 request_handler=PredefinedRequestHandler, 1981 protocol_version=protocol_version) 1982 # Bytes read and written by the server 1983 self.bytes_read = 0 1984 self.bytes_written = 0 1985 self.canned_response = None 1986 1987 1988class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer): 1989 pass 1990 1991 1992if features.HTTPSServerFeature.available(): 1993 from . import https_server 1994 1995 class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer): 1996 pass 1997 1998 1999class TestActivityMixin(object): 2000 """Test socket activity reporting. 2001 2002 We use a special purpose server to control the bytes sent and received and 2003 be able to predict the activity on the client socket. 2004 """ 2005 2006 def setUp(self): 2007 self.server = self._activity_server(self._protocol_version) 2008 self.server.start_server() 2009 self.addCleanup(self.server.stop_server) 2010 _activities = {} # Don't close over self and create a cycle 2011 2012 def report_activity(t, bytes, direction): 2013 count = _activities.get(direction, 0) 2014 count += bytes 2015 _activities[direction] = count 2016 self.activities = _activities 2017 # We override at class level because constructors may propagate the 2018 # bound method and render instance overriding ineffective (an 2019 # alternative would be to define a specific ui factory instead...) 2020 self.overrideAttr(self._transport, '_report_activity', report_activity) 2021 2022 def get_transport(self): 2023 t = self._transport(self.server.get_url()) 2024 # FIXME: Needs cleanup -- vila 20100611 2025 return t 2026 2027 def assertActivitiesMatch(self): 2028 self.assertEqual(self.server.bytes_read, 2029 self.activities.get('write', 0), 'written bytes') 2030 self.assertEqual(self.server.bytes_written, 2031 self.activities.get('read', 0), 'read bytes') 2032 2033 def test_get(self): 2034 self.server.canned_response = b'''HTTP/1.1 200 OK\r 2035Date: Tue, 11 Jul 2006 04:32:56 GMT\r 2036Server: Apache/2.0.54 (Fedora)\r 2037Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r 2038ETag: "56691-23-38e9ae00"\r 2039Accept-Ranges: bytes\r 2040Content-Length: 35\r 2041Connection: close\r 2042Content-Type: text/plain; charset=UTF-8\r 2043\r 2044Bazaar-NG meta directory, format 1 2045''' 2046 t = self.get_transport() 2047 self.assertEqual(b'Bazaar-NG meta directory, format 1\n', 2048 t.get('foo/bar').read()) 2049 self.assertActivitiesMatch() 2050 2051 def test_has(self): 2052 self.server.canned_response = b'''HTTP/1.1 200 OK\r 2053Server: SimpleHTTP/0.6 Python/2.5.2\r 2054Date: Thu, 29 Jan 2009 20:21:47 GMT\r 2055Content-type: application/octet-stream\r 2056Content-Length: 20\r 2057Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r 2058\r 2059''' 2060 t = self.get_transport() 2061 self.assertTrue(t.has('foo/bar')) 2062 self.assertActivitiesMatch() 2063 2064 def test_readv(self): 2065 self.server.canned_response = b'''HTTP/1.1 206 Partial Content\r 2066Date: Tue, 11 Jul 2006 04:49:48 GMT\r 2067Server: Apache/2.0.54 (Fedora)\r 2068Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r 2069ETag: "238a3c-16ec2-805c5540"\r 2070Accept-Ranges: bytes\r 2071Content-Length: 1534\r 2072Connection: close\r 2073Content-Type: multipart/byteranges; boundary=418470f848b63279b\r 2074\r 2075\r 2076--418470f848b63279b\r 2077Content-type: text/plain; charset=UTF-8\r 2078Content-range: bytes 0-254/93890\r 2079\r 2080mbp@sourcefrog.net-20050309040815-13242001617e4a06 2081mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627 2082mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8 2083mbp@sourcefrog.net-20050309041501-c840e09071de3b67 2084mbp@sourcefrog.net-20050309044615-c24a3250be83220a 2085\r 2086--418470f848b63279b\r 2087Content-type: text/plain; charset=UTF-8\r 2088Content-range: bytes 1000-2049/93890\r 2089\r 209040-fd4ec249b6b139ab 2091mbp@sourcefrog.net-20050311063625-07858525021f270b 2092mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9 2093mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a 2094mbp@sourcefrog.net-20050311232353-f5e33da490872c6a 2095mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0 2096mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb 2097mbp@sourcefrog.net-20050312073831-a47c3335ece1920f 2098mbp@sourcefrog.net-20050312085412-13373aa129ccbad3 2099mbp@sourcefrog.net-20050313052251-2bf004cb96b39933 2100mbp@sourcefrog.net-20050313052856-3edd84094687cb11 2101mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d 2102mbp@sourcefrog.net-20050313053853-7c64085594ff3072 2103mbp@sourcefrog.net-20050313054757-a86c3f5871069e22 2104mbp@sourcefrog.net-20050313061422-418f1f73b94879b9 2105mbp@sourcefrog.net-20050313120651-497bd231b19df600 2106mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a 2107mbp@sourcefrog.net-20050314025438-d52099f915fe65fc 2108mbp@sourcefrog.net-20050314025539-637a636692c055cf 2109mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba 2110mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62 2111mbp@source\r 2112--418470f848b63279b--\r 2113''' 2114 t = self.get_transport() 2115 # Remember that the request is ignored and that the ranges below 2116 # doesn't have to match the canned response. 2117 l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050)))) 2118 # Force consumption of the last bytesrange boundary 2119 t._get_connection().cleanup_pipe() 2120 self.assertEqual(2, len(l)) 2121 self.assertActivitiesMatch() 2122 2123 def test_post(self): 2124 self.server.canned_response = b'''HTTP/1.1 200 OK\r 2125Date: Tue, 11 Jul 2006 04:32:56 GMT\r 2126Server: Apache/2.0.54 (Fedora)\r 2127Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r 2128ETag: "56691-23-38e9ae00"\r 2129Accept-Ranges: bytes\r 2130Content-Length: 35\r 2131Connection: close\r 2132Content-Type: text/plain; charset=UTF-8\r 2133\r 2134lalala whatever as long as itsssss 2135''' 2136 t = self.get_transport() 2137 # We must send a single line of body bytes, see 2138 # PredefinedRequestHandler._handle_one_request 2139 code, f = t._post(b'abc def end-of-body\n') 2140 self.assertEqual(b'lalala whatever as long as itsssss\n', f.read()) 2141 self.assertActivitiesMatch() 2142 2143 2144class TestActivity(tests.TestCase, TestActivityMixin): 2145 2146 scenarios = multiply_scenarios( 2147 vary_by_http_activity(), 2148 vary_by_http_protocol_version(), 2149 ) 2150 2151 def setUp(self): 2152 super(TestActivity, self).setUp() 2153 TestActivityMixin.setUp(self) 2154 2155 2156class TestNoReportActivity(tests.TestCase, TestActivityMixin): 2157 2158 # Unlike TestActivity, we are really testing ReportingFileSocket and 2159 # ReportingSocket, so we don't need all the parametrization. Since 2160 # ReportingFileSocket and ReportingSocket are wrappers, it's easier to 2161 # test them through their use by the transport than directly (that's a 2162 # bit less clean but far more simpler and effective). 2163 _activity_server = ActivityHTTPServer 2164 _protocol_version = 'HTTP/1.1' 2165 2166 def setUp(self): 2167 super(TestNoReportActivity, self).setUp() 2168 self._transport = HttpTransport 2169 TestActivityMixin.setUp(self) 2170 2171 def assertActivitiesMatch(self): 2172 # Nothing to check here 2173 pass 2174 2175 2176class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver): 2177 """Test authentication on the redirected http server.""" 2178 2179 scenarios = vary_by_http_protocol_version() 2180 2181 _auth_header = 'Authorization' 2182 _password_prompt_prefix = '' 2183 _username_prompt_prefix = '' 2184 _auth_server = http_utils.HTTPBasicAuthServer 2185 _transport = HttpTransport 2186 2187 def setUp(self): 2188 super(TestAuthOnRedirected, self).setUp() 2189 self.build_tree_contents([('a', b'a'), 2190 ('1/',), 2191 ('1/a', b'redirected once'), 2192 ],) 2193 new_prefix = 'http://%s:%s' % (self.new_server.host, 2194 self.new_server.port) 2195 self.old_server.redirections = [ 2196 ('(.*)', r'%s/1\1' % (new_prefix), 301), ] 2197 self.old_transport = self.get_old_transport() 2198 self.new_server.add_user('joe', 'foo') 2199 cleanup_http_redirection_connections(self) 2200 2201 def create_transport_readonly_server(self): 2202 server = self._auth_server(protocol_version=self._protocol_version) 2203 server._url_protocol = self._url_protocol 2204 return server 2205 2206 def get_a(self, t): 2207 return t.get('a') 2208 2209 def test_auth_on_redirected_via_do_catching_redirections(self): 2210 self.redirections = 0 2211 2212 def redirected(t, exception, redirection_notice): 2213 self.redirections += 1 2214 redirected_t = t._redirected_to(exception.source, exception.target) 2215 self.addCleanup(redirected_t.disconnect) 2216 return redirected_t 2217 2218 ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n') 2219 self.assertEqual(b'redirected once', 2220 transport.do_catching_redirections( 2221 self.get_a, self.old_transport, redirected).read()) 2222 self.assertEqual(1, self.redirections) 2223 # stdin should be empty 2224 self.assertEqual('', ui.ui_factory.stdin.readline()) 2225 # stdout should be empty, stderr will contains the prompts 2226 self.assertEqual('', ui.ui_factory.stdout.getvalue()) 2227 2228 def test_auth_on_redirected_via_following_redirections(self): 2229 self.new_server.add_user('joe', 'foo') 2230 ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n') 2231 t = self.old_transport 2232 new_prefix = 'http://%s:%s' % (self.new_server.host, 2233 self.new_server.port) 2234 self.old_server.redirections = [ 2235 ('(.*)', r'%s/1\1' % (new_prefix), 301), ] 2236 self.assertEqual( 2237 b'redirected once', 2238 t.request('GET', t.abspath('a'), retries=3).read()) 2239 # stdin should be empty 2240 self.assertEqual('', ui.ui_factory.stdin.readline()) 2241 # stdout should be empty, stderr will contains the prompts 2242 self.assertEqual('', ui.ui_factory.stdout.getvalue()) 2243