1"""Tests for TCP connection handling, including proper and timely close.""" 2 3from __future__ import absolute_import, division, print_function 4__metaclass__ = type 5 6import errno 7import socket 8import time 9import logging 10import traceback as traceback_ 11from collections import namedtuple 12 13from six.moves import range, http_client, urllib 14 15import six 16import pytest 17from jaraco.text import trim, unwrap 18 19from cheroot.test import helper, webtest 20from cheroot._compat import IS_CI, IS_PYPY, IS_WINDOWS 21import cheroot.server 22 23 24timeout = 1 25pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN' 26 27 28class Controller(helper.Controller): 29 """Controller for serving WSGI apps.""" 30 31 def hello(req, resp): 32 """Render Hello world.""" 33 return 'Hello, world!' 34 35 def pov(req, resp): 36 """Render ``pov`` value.""" 37 return pov 38 39 def stream(req, resp): 40 """Render streaming response.""" 41 if 'set_cl' in req.environ['QUERY_STRING']: 42 resp.headers['Content-Length'] = str(10) 43 44 def content(): 45 for x in range(10): 46 yield str(x) 47 48 return content() 49 50 def upload(req, resp): 51 """Process file upload and render thank.""" 52 if not req.environ['REQUEST_METHOD'] == 'POST': 53 raise AssertionError( 54 "'POST' != request.method %r" % 55 req.environ['REQUEST_METHOD'], 56 ) 57 return "thanks for '%s'" % req.environ['wsgi.input'].read() 58 59 def custom_204(req, resp): 60 """Render response with status 204.""" 61 resp.status = '204' 62 return 'Code = 204' 63 64 def custom_304(req, resp): 65 """Render response with status 304.""" 66 resp.status = '304' 67 return 'Code = 304' 68 69 def err_before_read(req, resp): 70 """Render response with status 500.""" 71 resp.status = '500 Internal Server Error' 72 return 'ok' 73 74 def one_megabyte_of_a(req, resp): 75 """Render 1MB response.""" 76 return ['a' * 1024] * 1024 77 78 def wrong_cl_buffered(req, resp): 79 """Render buffered response with invalid length value.""" 80 resp.headers['Content-Length'] = '5' 81 return 'I have too many bytes' 82 83 def wrong_cl_unbuffered(req, resp): 84 """Render unbuffered response with invalid length value.""" 85 resp.headers['Content-Length'] = '5' 86 return ['I too', ' have too many bytes'] 87 88 def _munge(string): 89 """Encode PATH_INFO correctly depending on Python version. 90 91 WSGI 1.0 is a mess around unicode. Create endpoints 92 that match the PATH_INFO that it produces. 93 """ 94 if six.PY2: 95 return string 96 return string.encode('utf-8').decode('latin-1') 97 98 handlers = { 99 '/hello': hello, 100 '/pov': pov, 101 '/page1': pov, 102 '/page2': pov, 103 '/page3': pov, 104 '/stream': stream, 105 '/upload': upload, 106 '/custom/204': custom_204, 107 '/custom/304': custom_304, 108 '/err_before_read': err_before_read, 109 '/one_megabyte_of_a': one_megabyte_of_a, 110 '/wrong_cl_buffered': wrong_cl_buffered, 111 '/wrong_cl_unbuffered': wrong_cl_unbuffered, 112 } 113 114 115class ErrorLogMonitor: 116 """Mock class to access the server error_log calls made by the server.""" 117 118 ErrorLogCall = namedtuple('ErrorLogCall', ['msg', 'level', 'traceback']) 119 120 def __init__(self): 121 """Initialize the server error log monitor/interceptor. 122 123 If you need to ignore a particular error message use the property 124 ``ignored_msgs`` by appending to the list the expected error messages. 125 """ 126 self.calls = [] 127 # to be used the the teardown validation 128 self.ignored_msgs = [] 129 130 def __call__(self, msg='', level=logging.INFO, traceback=False): 131 """Intercept the call to the server error_log method.""" 132 if traceback: 133 tblines = traceback_.format_exc() 134 else: 135 tblines = '' 136 self.calls.append(ErrorLogMonitor.ErrorLogCall(msg, level, tblines)) 137 138 139@pytest.fixture 140def raw_testing_server(wsgi_server_client): 141 """Attach a WSGI app to the given server and preconfigure it.""" 142 app = Controller() 143 144 def _timeout(req, resp): 145 return str(wsgi_server.timeout) 146 app.handlers['/timeout'] = _timeout 147 wsgi_server = wsgi_server_client.server_instance 148 wsgi_server.wsgi_app = app 149 wsgi_server.max_request_body_size = 1001 150 wsgi_server.timeout = timeout 151 wsgi_server.server_client = wsgi_server_client 152 wsgi_server.keep_alive_conn_limit = 2 153 154 return wsgi_server 155 156 157@pytest.fixture 158def testing_server(raw_testing_server, monkeypatch): 159 """Modify the "raw" base server to monitor the error_log messages. 160 161 If you need to ignore a particular error message use the property 162 ``testing_server.error_log.ignored_msgs`` by appending to the list 163 the expected error messages. 164 """ 165 # patch the error_log calls of the server instance 166 monkeypatch.setattr(raw_testing_server, 'error_log', ErrorLogMonitor()) 167 168 yield raw_testing_server 169 170 # Teardown verification, in case that the server logged an 171 # error that wasn't notified to the client or we just made a mistake. 172 for c_msg, c_level, c_traceback in raw_testing_server.error_log.calls: 173 if c_level <= logging.WARNING: 174 continue 175 176 assert c_msg in raw_testing_server.error_log.ignored_msgs, ( 177 'Found error in the error log: ' 178 "message = '{c_msg}', level = '{c_level}'\n" 179 '{c_traceback}'.format(**locals()), 180 ) 181 182 183@pytest.fixture 184def test_client(testing_server): 185 """Get and return a test client out of the given server.""" 186 return testing_server.server_client 187 188 189def header_exists(header_name, headers): 190 """Check that a header is present.""" 191 return header_name.lower() in (k.lower() for (k, _) in headers) 192 193 194def header_has_value(header_name, header_value, headers): 195 """Check that a header with a given value is present.""" 196 return header_name.lower() in ( 197 k.lower() for (k, v) in headers 198 if v == header_value 199 ) 200 201 202def test_HTTP11_persistent_connections(test_client): 203 """Test persistent HTTP/1.1 connections.""" 204 # Initialize a persistent HTTP connection 205 http_connection = test_client.get_connection() 206 http_connection.auto_open = False 207 http_connection.connect() 208 209 # Make the first request and assert there's no "Connection: close". 210 status_line, actual_headers, actual_resp_body = test_client.get( 211 '/pov', http_conn=http_connection, 212 ) 213 actual_status = int(status_line[:3]) 214 assert actual_status == 200 215 assert status_line[4:] == 'OK' 216 assert actual_resp_body == pov.encode() 217 assert not header_exists('Connection', actual_headers) 218 219 # Make another request on the same connection. 220 status_line, actual_headers, actual_resp_body = test_client.get( 221 '/page1', http_conn=http_connection, 222 ) 223 actual_status = int(status_line[:3]) 224 assert actual_status == 200 225 assert status_line[4:] == 'OK' 226 assert actual_resp_body == pov.encode() 227 assert not header_exists('Connection', actual_headers) 228 229 # Test client-side close. 230 status_line, actual_headers, actual_resp_body = test_client.get( 231 '/page2', http_conn=http_connection, 232 headers=[('Connection', 'close')], 233 ) 234 actual_status = int(status_line[:3]) 235 assert actual_status == 200 236 assert status_line[4:] == 'OK' 237 assert actual_resp_body == pov.encode() 238 assert header_has_value('Connection', 'close', actual_headers) 239 240 # Make another request on the same connection, which should error. 241 with pytest.raises(http_client.NotConnected): 242 test_client.get('/pov', http_conn=http_connection) 243 244 245@pytest.mark.parametrize( 246 'set_cl', 247 ( 248 False, # Without Content-Length 249 True, # With Content-Length 250 ), 251) 252def test_streaming_11(test_client, set_cl): 253 """Test serving of streaming responses with HTTP/1.1 protocol.""" 254 # Initialize a persistent HTTP connection 255 http_connection = test_client.get_connection() 256 http_connection.auto_open = False 257 http_connection.connect() 258 259 # Make the first request and assert there's no "Connection: close". 260 status_line, actual_headers, actual_resp_body = test_client.get( 261 '/pov', http_conn=http_connection, 262 ) 263 actual_status = int(status_line[:3]) 264 assert actual_status == 200 265 assert status_line[4:] == 'OK' 266 assert actual_resp_body == pov.encode() 267 assert not header_exists('Connection', actual_headers) 268 269 # Make another, streamed request on the same connection. 270 if set_cl: 271 # When a Content-Length is provided, the content should stream 272 # without closing the connection. 273 status_line, actual_headers, actual_resp_body = test_client.get( 274 '/stream?set_cl=Yes', http_conn=http_connection, 275 ) 276 assert header_exists('Content-Length', actual_headers) 277 assert not header_has_value('Connection', 'close', actual_headers) 278 assert not header_exists('Transfer-Encoding', actual_headers) 279 280 assert actual_status == 200 281 assert status_line[4:] == 'OK' 282 assert actual_resp_body == b'0123456789' 283 else: 284 # When no Content-Length response header is provided, 285 # streamed output will either close the connection, or use 286 # chunked encoding, to determine transfer-length. 287 status_line, actual_headers, actual_resp_body = test_client.get( 288 '/stream', http_conn=http_connection, 289 ) 290 assert not header_exists('Content-Length', actual_headers) 291 assert actual_status == 200 292 assert status_line[4:] == 'OK' 293 assert actual_resp_body == b'0123456789' 294 295 chunked_response = False 296 for k, v in actual_headers: 297 if k.lower() == 'transfer-encoding': 298 if str(v) == 'chunked': 299 chunked_response = True 300 301 if chunked_response: 302 assert not header_has_value('Connection', 'close', actual_headers) 303 else: 304 assert header_has_value('Connection', 'close', actual_headers) 305 306 # Make another request on the same connection, which should 307 # error. 308 with pytest.raises(http_client.NotConnected): 309 test_client.get('/pov', http_conn=http_connection) 310 311 # Try HEAD. 312 # See https://www.bitbucket.org/cherrypy/cherrypy/issue/864. 313 # TODO: figure out how can this be possible on an closed connection 314 # (chunked_response case) 315 status_line, actual_headers, actual_resp_body = test_client.head( 316 '/stream', http_conn=http_connection, 317 ) 318 assert actual_status == 200 319 assert status_line[4:] == 'OK' 320 assert actual_resp_body == b'' 321 assert not header_exists('Transfer-Encoding', actual_headers) 322 323 324@pytest.mark.parametrize( 325 'set_cl', 326 ( 327 False, # Without Content-Length 328 True, # With Content-Length 329 ), 330) 331def test_streaming_10(test_client, set_cl): 332 """Test serving of streaming responses with HTTP/1.0 protocol.""" 333 original_server_protocol = test_client.server_instance.protocol 334 test_client.server_instance.protocol = 'HTTP/1.0' 335 336 # Initialize a persistent HTTP connection 337 http_connection = test_client.get_connection() 338 http_connection.auto_open = False 339 http_connection.connect() 340 341 # Make the first request and assert Keep-Alive. 342 status_line, actual_headers, actual_resp_body = test_client.get( 343 '/pov', http_conn=http_connection, 344 headers=[('Connection', 'Keep-Alive')], 345 protocol='HTTP/1.0', 346 ) 347 actual_status = int(status_line[:3]) 348 assert actual_status == 200 349 assert status_line[4:] == 'OK' 350 assert actual_resp_body == pov.encode() 351 assert header_has_value('Connection', 'Keep-Alive', actual_headers) 352 353 # Make another, streamed request on the same connection. 354 if set_cl: 355 # When a Content-Length is provided, the content should 356 # stream without closing the connection. 357 status_line, actual_headers, actual_resp_body = test_client.get( 358 '/stream?set_cl=Yes', http_conn=http_connection, 359 headers=[('Connection', 'Keep-Alive')], 360 protocol='HTTP/1.0', 361 ) 362 actual_status = int(status_line[:3]) 363 assert actual_status == 200 364 assert status_line[4:] == 'OK' 365 assert actual_resp_body == b'0123456789' 366 367 assert header_exists('Content-Length', actual_headers) 368 assert header_has_value('Connection', 'Keep-Alive', actual_headers) 369 assert not header_exists('Transfer-Encoding', actual_headers) 370 else: 371 # When a Content-Length is not provided, 372 # the server should close the connection. 373 status_line, actual_headers, actual_resp_body = test_client.get( 374 '/stream', http_conn=http_connection, 375 headers=[('Connection', 'Keep-Alive')], 376 protocol='HTTP/1.0', 377 ) 378 actual_status = int(status_line[:3]) 379 assert actual_status == 200 380 assert status_line[4:] == 'OK' 381 assert actual_resp_body == b'0123456789' 382 383 assert not header_exists('Content-Length', actual_headers) 384 assert not header_has_value('Connection', 'Keep-Alive', actual_headers) 385 assert not header_exists('Transfer-Encoding', actual_headers) 386 387 # Make another request on the same connection, which should error. 388 with pytest.raises(http_client.NotConnected): 389 test_client.get( 390 '/pov', http_conn=http_connection, 391 protocol='HTTP/1.0', 392 ) 393 394 test_client.server_instance.protocol = original_server_protocol 395 396 397@pytest.mark.parametrize( 398 'http_server_protocol', 399 ( 400 'HTTP/1.0', 401 pytest.param( 402 'HTTP/1.1', 403 marks=pytest.mark.xfail( 404 IS_PYPY and IS_CI, 405 reason='Fails under PyPy in CI for unknown reason', 406 strict=False, 407 ), 408 ), 409 ), 410) 411def test_keepalive(test_client, http_server_protocol): 412 """Test Keep-Alive enabled connections.""" 413 original_server_protocol = test_client.server_instance.protocol 414 test_client.server_instance.protocol = http_server_protocol 415 416 http_client_protocol = 'HTTP/1.0' 417 418 # Initialize a persistent HTTP connection 419 http_connection = test_client.get_connection() 420 http_connection.auto_open = False 421 http_connection.connect() 422 423 # Test a normal HTTP/1.0 request. 424 status_line, actual_headers, actual_resp_body = test_client.get( 425 '/page2', 426 protocol=http_client_protocol, 427 ) 428 actual_status = int(status_line[:3]) 429 assert actual_status == 200 430 assert status_line[4:] == 'OK' 431 assert actual_resp_body == pov.encode() 432 assert not header_exists('Connection', actual_headers) 433 434 # Test a keep-alive HTTP/1.0 request. 435 436 status_line, actual_headers, actual_resp_body = test_client.get( 437 '/page3', headers=[('Connection', 'Keep-Alive')], 438 http_conn=http_connection, protocol=http_client_protocol, 439 ) 440 actual_status = int(status_line[:3]) 441 assert actual_status == 200 442 assert status_line[4:] == 'OK' 443 assert actual_resp_body == pov.encode() 444 assert header_has_value('Connection', 'Keep-Alive', actual_headers) 445 assert header_has_value( 446 'Keep-Alive', 447 'timeout={test_client.server_instance.timeout}'.format(**locals()), 448 actual_headers, 449 ) 450 451 # Remove the keep-alive header again. 452 status_line, actual_headers, actual_resp_body = test_client.get( 453 '/page3', http_conn=http_connection, 454 protocol=http_client_protocol, 455 ) 456 actual_status = int(status_line[:3]) 457 assert actual_status == 200 458 assert status_line[4:] == 'OK' 459 assert actual_resp_body == pov.encode() 460 assert not header_exists('Connection', actual_headers) 461 assert not header_exists('Keep-Alive', actual_headers) 462 463 test_client.server_instance.protocol = original_server_protocol 464 465 466def test_keepalive_conn_management(test_client): 467 """Test management of Keep-Alive connections.""" 468 test_client.server_instance.timeout = 2 469 470 def connection(): 471 # Initialize a persistent HTTP connection 472 http_connection = test_client.get_connection() 473 http_connection.auto_open = False 474 http_connection.connect() 475 return http_connection 476 477 def request(conn, keepalive=True): 478 status_line, actual_headers, actual_resp_body = test_client.get( 479 '/page3', headers=[('Connection', 'Keep-Alive')], 480 http_conn=conn, protocol='HTTP/1.0', 481 ) 482 actual_status = int(status_line[:3]) 483 assert actual_status == 200 484 assert status_line[4:] == 'OK' 485 assert actual_resp_body == pov.encode() 486 if keepalive: 487 assert header_has_value('Connection', 'Keep-Alive', actual_headers) 488 assert header_has_value( 489 'Keep-Alive', 490 'timeout={test_client.server_instance.timeout}'. 491 format(**locals()), 492 actual_headers, 493 ) 494 else: 495 assert not header_exists('Connection', actual_headers) 496 assert not header_exists('Keep-Alive', actual_headers) 497 498 def check_server_idle_conn_count(count, timeout=1.0): 499 deadline = time.time() + timeout 500 while True: 501 n = test_client.server_instance._connections._num_connections 502 if n == count: 503 return 504 assert time.time() <= deadline, ( 505 'idle conn count mismatch, wanted {count}, got {n}'. 506 format(**locals()), 507 ) 508 509 disconnect_errors = ( 510 http_client.BadStatusLine, 511 http_client.CannotSendRequest, 512 http_client.NotConnected, 513 ) 514 515 # Make a new connection. 516 c1 = connection() 517 request(c1) 518 check_server_idle_conn_count(1) 519 520 # Make a second one. 521 c2 = connection() 522 request(c2) 523 check_server_idle_conn_count(2) 524 525 # Reusing the first connection should still work. 526 request(c1) 527 check_server_idle_conn_count(2) 528 529 # Creating a new connection should still work, but we should 530 # have run out of available connections to keep alive, so the 531 # server should tell us to close. 532 c3 = connection() 533 request(c3, keepalive=False) 534 check_server_idle_conn_count(2) 535 536 # Show that the third connection was closed. 537 with pytest.raises(disconnect_errors): 538 request(c3) 539 check_server_idle_conn_count(2) 540 541 # Wait for some of our timeout. 542 time.sleep(1.2) 543 544 # Refresh the second connection. 545 request(c2) 546 check_server_idle_conn_count(2) 547 548 # Wait for the remainder of our timeout, plus one tick. 549 time.sleep(1.2) 550 check_server_idle_conn_count(1) 551 552 # First connection should now be expired. 553 with pytest.raises(disconnect_errors): 554 request(c1) 555 check_server_idle_conn_count(1) 556 557 # But the second one should still be valid. 558 request(c2) 559 check_server_idle_conn_count(1) 560 561 # Restore original timeout. 562 test_client.server_instance.timeout = timeout 563 564 565@pytest.mark.parametrize( 566 ('simulated_exception', 'error_number', 'exception_leaks'), 567 ( 568 pytest.param( 569 socket.error, errno.ECONNRESET, False, 570 id='socket.error(ECONNRESET)', 571 ), 572 pytest.param( 573 socket.error, errno.EPIPE, False, 574 id='socket.error(EPIPE)', 575 ), 576 pytest.param( 577 socket.error, errno.ENOTCONN, False, 578 id='simulated socket.error(ENOTCONN)', 579 ), 580 pytest.param( 581 None, # <-- don't raise an artificial exception 582 errno.ENOTCONN, False, 583 id='real socket.error(ENOTCONN)', 584 marks=pytest.mark.xfail( 585 IS_WINDOWS, 586 reason='Now reproducible this way on Windows', 587 ), 588 ), 589 pytest.param( 590 socket.error, errno.ESHUTDOWN, False, 591 id='socket.error(ESHUTDOWN)', 592 ), 593 pytest.param(RuntimeError, 666, True, id='RuntimeError(666)'), 594 pytest.param(socket.error, -1, True, id='socket.error(-1)'), 595 ) + ( 596 () if six.PY2 else ( 597 pytest.param( 598 ConnectionResetError, errno.ECONNRESET, False, 599 id='ConnectionResetError(ECONNRESET)', 600 ), 601 pytest.param( 602 BrokenPipeError, errno.EPIPE, False, 603 id='BrokenPipeError(EPIPE)', 604 ), 605 pytest.param( 606 BrokenPipeError, errno.ESHUTDOWN, False, 607 id='BrokenPipeError(ESHUTDOWN)', 608 ), 609 ) 610 ), 611) 612def test_broken_connection_during_tcp_fin( 613 error_number, exception_leaks, 614 mocker, monkeypatch, 615 simulated_exception, test_client, 616): 617 """Test there's no traceback on broken connection during close. 618 619 It artificially causes :py:data:`~errno.ECONNRESET` / 620 :py:data:`~errno.EPIPE` / :py:data:`~errno.ESHUTDOWN` / 621 :py:data:`~errno.ENOTCONN` as well as unrelated :py:exc:`RuntimeError` 622 and :py:exc:`socket.error(-1) <socket.error>` on the server socket when 623 :py:meth:`socket.shutdown() <socket.socket.shutdown>` is called. It's 624 triggered by closing the client socket before the server had a chance 625 to respond. 626 627 The expectation is that only :py:exc:`RuntimeError` and a 628 :py:exc:`socket.error` with an unusual error code would leak. 629 630 With the :py:data:`None`-parameter, a real non-simulated 631 :py:exc:`OSError(107, 'Transport endpoint is not connected') 632 <OSError>` happens. 633 """ 634 exc_instance = ( 635 None if simulated_exception is None 636 else simulated_exception(error_number, 'Simulated socket error') 637 ) 638 old_close_kernel_socket = ( 639 test_client.server_instance. 640 ConnectionClass._close_kernel_socket 641 ) 642 643 def _close_kernel_socket(self): 644 monkeypatch.setattr( # `socket.shutdown` is read-only otherwise 645 self, 'socket', 646 mocker.mock_module.Mock(wraps=self.socket), 647 ) 648 if exc_instance is not None: 649 monkeypatch.setattr( 650 self.socket, 'shutdown', 651 mocker.mock_module.Mock(side_effect=exc_instance), 652 ) 653 _close_kernel_socket.fin_spy = mocker.spy(self.socket, 'shutdown') 654 _close_kernel_socket.exception_leaked = True 655 old_close_kernel_socket(self) 656 _close_kernel_socket.exception_leaked = False 657 658 monkeypatch.setattr( 659 test_client.server_instance.ConnectionClass, 660 '_close_kernel_socket', 661 _close_kernel_socket, 662 ) 663 664 conn = test_client.get_connection() 665 conn.auto_open = False 666 conn.connect() 667 conn.send(b'GET /hello HTTP/1.1') 668 conn.send(('Host: %s' % conn.host).encode('ascii')) 669 conn.close() 670 671 for _ in range(10): # Let the server attempt TCP shutdown 672 time.sleep(0.1) 673 if hasattr(_close_kernel_socket, 'exception_leaked'): 674 break 675 676 if exc_instance is not None: # simulated by us 677 assert _close_kernel_socket.fin_spy.spy_exception is exc_instance 678 else: # real 679 assert isinstance( 680 _close_kernel_socket.fin_spy.spy_exception, socket.error, 681 ) 682 assert _close_kernel_socket.fin_spy.spy_exception.errno == error_number 683 684 assert _close_kernel_socket.exception_leaked is exception_leaks 685 686 687@pytest.mark.parametrize( 688 'timeout_before_headers', 689 ( 690 True, 691 False, 692 ), 693) 694def test_HTTP11_Timeout(test_client, timeout_before_headers): 695 """Check timeout without sending any data. 696 697 The server will close the connection with a 408. 698 """ 699 conn = test_client.get_connection() 700 conn.auto_open = False 701 conn.connect() 702 703 if not timeout_before_headers: 704 # Connect but send half the headers only. 705 conn.send(b'GET /hello HTTP/1.1') 706 conn.send(('Host: %s' % conn.host).encode('ascii')) 707 # else: Connect but send nothing. 708 709 # Wait for our socket timeout 710 time.sleep(timeout * 2) 711 712 # The request should have returned 408 already. 713 response = conn.response_class(conn.sock, method='GET') 714 response.begin() 715 assert response.status == 408 716 conn.close() 717 718 719def test_HTTP11_Timeout_after_request(test_client): 720 """Check timeout after at least one request has succeeded. 721 722 The server should close the connection without 408. 723 """ 724 fail_msg = "Writing to timed out socket didn't fail as it should have: %s" 725 726 # Make an initial request 727 conn = test_client.get_connection() 728 conn.putrequest('GET', '/timeout?t=%s' % timeout, skip_host=True) 729 conn.putheader('Host', conn.host) 730 conn.endheaders() 731 response = conn.response_class(conn.sock, method='GET') 732 response.begin() 733 assert response.status == 200 734 actual_body = response.read() 735 expected_body = str(timeout).encode() 736 assert actual_body == expected_body 737 738 # Make a second request on the same socket 739 conn._output(b'GET /hello HTTP/1.1') 740 conn._output(('Host: %s' % conn.host).encode('ascii')) 741 conn._send_output() 742 response = conn.response_class(conn.sock, method='GET') 743 response.begin() 744 assert response.status == 200 745 actual_body = response.read() 746 expected_body = b'Hello, world!' 747 assert actual_body == expected_body 748 749 # Wait for our socket timeout 750 time.sleep(timeout * 2) 751 752 # Make another request on the same socket, which should error 753 conn._output(b'GET /hello HTTP/1.1') 754 conn._output(('Host: %s' % conn.host).encode('ascii')) 755 conn._send_output() 756 response = conn.response_class(conn.sock, method='GET') 757 try: 758 response.begin() 759 except (socket.error, http_client.BadStatusLine): 760 pass 761 except Exception as ex: 762 pytest.fail(fail_msg % ex) 763 else: 764 if response.status != 408: 765 pytest.fail(fail_msg % response.read()) 766 767 conn.close() 768 769 # Make another request on a new socket, which should work 770 conn = test_client.get_connection() 771 conn.putrequest('GET', '/pov', skip_host=True) 772 conn.putheader('Host', conn.host) 773 conn.endheaders() 774 response = conn.response_class(conn.sock, method='GET') 775 response.begin() 776 assert response.status == 200 777 actual_body = response.read() 778 expected_body = pov.encode() 779 assert actual_body == expected_body 780 781 # Make another request on the same socket, 782 # but timeout on the headers 783 conn.send(b'GET /hello HTTP/1.1') 784 # Wait for our socket timeout 785 time.sleep(timeout * 2) 786 response = conn.response_class(conn.sock, method='GET') 787 try: 788 response.begin() 789 except (socket.error, http_client.BadStatusLine): 790 pass 791 except Exception as ex: 792 pytest.fail(fail_msg % ex) 793 else: 794 if response.status != 408: 795 pytest.fail(fail_msg % response.read()) 796 797 conn.close() 798 799 # Retry the request on a new connection, which should work 800 conn = test_client.get_connection() 801 conn.putrequest('GET', '/pov', skip_host=True) 802 conn.putheader('Host', conn.host) 803 conn.endheaders() 804 response = conn.response_class(conn.sock, method='GET') 805 response.begin() 806 assert response.status == 200 807 actual_body = response.read() 808 expected_body = pov.encode() 809 assert actual_body == expected_body 810 conn.close() 811 812 813def test_HTTP11_pipelining(test_client): 814 """Test HTTP/1.1 pipelining. 815 816 :py:mod:`http.client` doesn't support this directly. 817 """ 818 conn = test_client.get_connection() 819 820 # Put request 1 821 conn.putrequest('GET', '/hello', skip_host=True) 822 conn.putheader('Host', conn.host) 823 conn.endheaders() 824 825 for trial in range(5): 826 # Put next request 827 conn._output( 828 ('GET /hello?%s HTTP/1.1' % trial).encode('iso-8859-1'), 829 ) 830 conn._output(('Host: %s' % conn.host).encode('ascii')) 831 conn._send_output() 832 833 # Retrieve previous response 834 response = conn.response_class(conn.sock, method='GET') 835 # there is a bug in python3 regarding the buffering of 836 # ``conn.sock``. Until that bug get's fixed we will 837 # monkey patch the ``response`` instance. 838 # https://bugs.python.org/issue23377 839 if not six.PY2: 840 response.fp = conn.sock.makefile('rb', 0) 841 response.begin() 842 body = response.read(13) 843 assert response.status == 200 844 assert body == b'Hello, world!' 845 846 # Retrieve final response 847 response = conn.response_class(conn.sock, method='GET') 848 response.begin() 849 body = response.read() 850 assert response.status == 200 851 assert body == b'Hello, world!' 852 853 conn.close() 854 855 856def test_100_Continue(test_client): 857 """Test 100-continue header processing.""" 858 conn = test_client.get_connection() 859 860 # Try a page without an Expect request header first. 861 # Note that http.client's response.begin automatically ignores 862 # 100 Continue responses, so we must manually check for it. 863 conn.putrequest('POST', '/upload', skip_host=True) 864 conn.putheader('Host', conn.host) 865 conn.putheader('Content-Type', 'text/plain') 866 conn.putheader('Content-Length', '4') 867 conn.endheaders() 868 conn.send(b"d'oh") 869 response = conn.response_class(conn.sock, method='POST') 870 version, status, reason = response._read_status() 871 assert status != 100 872 conn.close() 873 874 # Now try a page with an Expect header... 875 conn.connect() 876 conn.putrequest('POST', '/upload', skip_host=True) 877 conn.putheader('Host', conn.host) 878 conn.putheader('Content-Type', 'text/plain') 879 conn.putheader('Content-Length', '17') 880 conn.putheader('Expect', '100-continue') 881 conn.endheaders() 882 response = conn.response_class(conn.sock, method='POST') 883 884 # ...assert and then skip the 100 response 885 version, status, reason = response._read_status() 886 assert status == 100 887 while True: 888 line = response.fp.readline().strip() 889 if line: 890 pytest.fail( 891 '100 Continue should not output any headers. Got %r' % 892 line, 893 ) 894 else: 895 break 896 897 # ...send the body 898 body = b'I am a small file' 899 conn.send(body) 900 901 # ...get the final response 902 response.begin() 903 status_line, actual_headers, actual_resp_body = webtest.shb(response) 904 actual_status = int(status_line[:3]) 905 assert actual_status == 200 906 expected_resp_body = ("thanks for '%s'" % body).encode() 907 assert actual_resp_body == expected_resp_body 908 conn.close() 909 910 911@pytest.mark.parametrize( 912 'max_request_body_size', 913 ( 914 0, 915 1001, 916 ), 917) 918def test_readall_or_close(test_client, max_request_body_size): 919 """Test a max_request_body_size of 0 (the default) and 1001.""" 920 old_max = test_client.server_instance.max_request_body_size 921 922 test_client.server_instance.max_request_body_size = max_request_body_size 923 924 conn = test_client.get_connection() 925 926 # Get a POST page with an error 927 conn.putrequest('POST', '/err_before_read', skip_host=True) 928 conn.putheader('Host', conn.host) 929 conn.putheader('Content-Type', 'text/plain') 930 conn.putheader('Content-Length', '1000') 931 conn.putheader('Expect', '100-continue') 932 conn.endheaders() 933 response = conn.response_class(conn.sock, method='POST') 934 935 # ...assert and then skip the 100 response 936 version, status, reason = response._read_status() 937 assert status == 100 938 skip = True 939 while skip: 940 skip = response.fp.readline().strip() 941 942 # ...send the body 943 conn.send(b'x' * 1000) 944 945 # ...get the final response 946 response.begin() 947 status_line, actual_headers, actual_resp_body = webtest.shb(response) 948 actual_status = int(status_line[:3]) 949 assert actual_status == 500 950 951 # Now try a working page with an Expect header... 952 conn._output(b'POST /upload HTTP/1.1') 953 conn._output(('Host: %s' % conn.host).encode('ascii')) 954 conn._output(b'Content-Type: text/plain') 955 conn._output(b'Content-Length: 17') 956 conn._output(b'Expect: 100-continue') 957 conn._send_output() 958 response = conn.response_class(conn.sock, method='POST') 959 960 # ...assert and then skip the 100 response 961 version, status, reason = response._read_status() 962 assert status == 100 963 skip = True 964 while skip: 965 skip = response.fp.readline().strip() 966 967 # ...send the body 968 body = b'I am a small file' 969 conn.send(body) 970 971 # ...get the final response 972 response.begin() 973 status_line, actual_headers, actual_resp_body = webtest.shb(response) 974 actual_status = int(status_line[:3]) 975 assert actual_status == 200 976 expected_resp_body = ("thanks for '%s'" % body).encode() 977 assert actual_resp_body == expected_resp_body 978 conn.close() 979 980 test_client.server_instance.max_request_body_size = old_max 981 982 983def test_No_Message_Body(test_client): 984 """Test HTTP queries with an empty response body.""" 985 # Initialize a persistent HTTP connection 986 http_connection = test_client.get_connection() 987 http_connection.auto_open = False 988 http_connection.connect() 989 990 # Make the first request and assert there's no "Connection: close". 991 status_line, actual_headers, actual_resp_body = test_client.get( 992 '/pov', http_conn=http_connection, 993 ) 994 actual_status = int(status_line[:3]) 995 assert actual_status == 200 996 assert status_line[4:] == 'OK' 997 assert actual_resp_body == pov.encode() 998 assert not header_exists('Connection', actual_headers) 999 1000 # Make a 204 request on the same connection. 1001 status_line, actual_headers, actual_resp_body = test_client.get( 1002 '/custom/204', http_conn=http_connection, 1003 ) 1004 actual_status = int(status_line[:3]) 1005 assert actual_status == 204 1006 assert not header_exists('Content-Length', actual_headers) 1007 assert actual_resp_body == b'' 1008 assert not header_exists('Connection', actual_headers) 1009 1010 # Make a 304 request on the same connection. 1011 status_line, actual_headers, actual_resp_body = test_client.get( 1012 '/custom/304', http_conn=http_connection, 1013 ) 1014 actual_status = int(status_line[:3]) 1015 assert actual_status == 304 1016 assert not header_exists('Content-Length', actual_headers) 1017 assert actual_resp_body == b'' 1018 assert not header_exists('Connection', actual_headers) 1019 1020 1021@pytest.mark.xfail( 1022 reason=unwrap( 1023 trim(""" 1024 Headers from earlier request leak into the request 1025 line for a subsequent request, resulting in 400 1026 instead of 413. See cherrypy/cheroot#69 for details. 1027 """), 1028 ), 1029) 1030def test_Chunked_Encoding(test_client): 1031 """Test HTTP uploads with chunked transfer-encoding.""" 1032 # Initialize a persistent HTTP connection 1033 conn = test_client.get_connection() 1034 1035 # Try a normal chunked request (with extensions) 1036 body = ( 1037 b'8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n' 1038 b'Content-Type: application/json\r\n' 1039 b'\r\n' 1040 ) 1041 conn.putrequest('POST', '/upload', skip_host=True) 1042 conn.putheader('Host', conn.host) 1043 conn.putheader('Transfer-Encoding', 'chunked') 1044 conn.putheader('Trailer', 'Content-Type') 1045 # Note that this is somewhat malformed: 1046 # we shouldn't be sending Content-Length. 1047 # RFC 2616 says the server should ignore it. 1048 conn.putheader('Content-Length', '3') 1049 conn.endheaders() 1050 conn.send(body) 1051 response = conn.getresponse() 1052 status_line, actual_headers, actual_resp_body = webtest.shb(response) 1053 actual_status = int(status_line[:3]) 1054 assert actual_status == 200 1055 assert status_line[4:] == 'OK' 1056 expected_resp_body = ("thanks for '%s'" % b'xx\r\nxxxxyyyyy').encode() 1057 assert actual_resp_body == expected_resp_body 1058 1059 # Try a chunked request that exceeds server.max_request_body_size. 1060 # Note that the delimiters and trailer are included. 1061 body = b'\r\n'.join((b'3e3', b'x' * 995, b'0', b'', b'')) 1062 conn.putrequest('POST', '/upload', skip_host=True) 1063 conn.putheader('Host', conn.host) 1064 conn.putheader('Transfer-Encoding', 'chunked') 1065 conn.putheader('Content-Type', 'text/plain') 1066 # Chunked requests don't need a content-length 1067 # conn.putheader("Content-Length", len(body)) 1068 conn.endheaders() 1069 conn.send(body) 1070 response = conn.getresponse() 1071 status_line, actual_headers, actual_resp_body = webtest.shb(response) 1072 actual_status = int(status_line[:3]) 1073 assert actual_status == 413 1074 conn.close() 1075 1076 1077def test_Content_Length_in(test_client): 1078 """Try a non-chunked request where Content-Length exceeds limit. 1079 1080 (server.max_request_body_size). 1081 Assert error before body send. 1082 """ 1083 # Initialize a persistent HTTP connection 1084 conn = test_client.get_connection() 1085 1086 conn.putrequest('POST', '/upload', skip_host=True) 1087 conn.putheader('Host', conn.host) 1088 conn.putheader('Content-Type', 'text/plain') 1089 conn.putheader('Content-Length', '9999') 1090 conn.endheaders() 1091 response = conn.getresponse() 1092 status_line, actual_headers, actual_resp_body = webtest.shb(response) 1093 actual_status = int(status_line[:3]) 1094 assert actual_status == 413 1095 expected_resp_body = ( 1096 b'The entity sent with the request exceeds ' 1097 b'the maximum allowed bytes.' 1098 ) 1099 assert actual_resp_body == expected_resp_body 1100 conn.close() 1101 1102 1103def test_Content_Length_not_int(test_client): 1104 """Test that malicious Content-Length header returns 400.""" 1105 status_line, actual_headers, actual_resp_body = test_client.post( 1106 '/upload', 1107 headers=[ 1108 ('Content-Type', 'text/plain'), 1109 ('Content-Length', 'not-an-integer'), 1110 ], 1111 ) 1112 actual_status = int(status_line[:3]) 1113 1114 assert actual_status == 400 1115 assert actual_resp_body == b'Malformed Content-Length Header.' 1116 1117 1118@pytest.mark.parametrize( 1119 ('uri', 'expected_resp_status', 'expected_resp_body'), 1120 ( 1121 ( 1122 '/wrong_cl_buffered', 500, 1123 ( 1124 b'The requested resource returned more bytes than the ' 1125 b'declared Content-Length.' 1126 ), 1127 ), 1128 ('/wrong_cl_unbuffered', 200, b'I too'), 1129 ), 1130) 1131def test_Content_Length_out( 1132 test_client, 1133 uri, expected_resp_status, expected_resp_body, 1134): 1135 """Test response with Content-Length less than the response body. 1136 1137 (non-chunked response) 1138 """ 1139 conn = test_client.get_connection() 1140 conn.putrequest('GET', uri, skip_host=True) 1141 conn.putheader('Host', conn.host) 1142 conn.endheaders() 1143 1144 response = conn.getresponse() 1145 status_line, actual_headers, actual_resp_body = webtest.shb(response) 1146 actual_status = int(status_line[:3]) 1147 1148 assert actual_status == expected_resp_status 1149 assert actual_resp_body == expected_resp_body 1150 1151 conn.close() 1152 1153 # the server logs the exception that we had verified from the 1154 # client perspective. Tell the error_log verification that 1155 # it can ignore that message. 1156 test_client.server_instance.error_log.ignored_msgs.extend(( 1157 # Python 3.7+: 1158 "ValueError('Response body exceeds the declared Content-Length.')", 1159 # Python 2.7-3.6 (macOS?): 1160 "ValueError('Response body exceeds the declared Content-Length.',)", 1161 )) 1162 1163 1164@pytest.mark.xfail( 1165 reason='Sometimes this test fails due to low timeout. ' 1166 'Ref: https://github.com/cherrypy/cherrypy/issues/598', 1167) 1168def test_598(test_client): 1169 """Test serving large file with a read timeout in place.""" 1170 # Initialize a persistent HTTP connection 1171 conn = test_client.get_connection() 1172 remote_data_conn = urllib.request.urlopen( 1173 '%s://%s:%s/one_megabyte_of_a' 1174 % ('http', conn.host, conn.port), 1175 ) 1176 buf = remote_data_conn.read(512) 1177 time.sleep(timeout * 0.6) 1178 remaining = (1024 * 1024) - 512 1179 while remaining: 1180 data = remote_data_conn.read(remaining) 1181 if not data: 1182 break 1183 buf += data 1184 remaining -= len(data) 1185 1186 assert len(buf) == 1024 * 1024 1187 assert buf == b'a' * 1024 * 1024 1188 assert remaining == 0 1189 remote_data_conn.close() 1190 1191 1192@pytest.mark.parametrize( 1193 'invalid_terminator', 1194 ( 1195 b'\n\n', 1196 b'\r\n\n', 1197 ), 1198) 1199def test_No_CRLF(test_client, invalid_terminator): 1200 """Test HTTP queries with no valid CRLF terminators.""" 1201 # Initialize a persistent HTTP connection 1202 conn = test_client.get_connection() 1203 1204 # (b'%s' % b'') is not supported in Python 3.4, so just use bytes.join() 1205 conn.send(b''.join((b'GET /hello HTTP/1.1', invalid_terminator))) 1206 response = conn.response_class(conn.sock, method='GET') 1207 response.begin() 1208 actual_resp_body = response.read() 1209 expected_resp_body = b'HTTP requires CRLF terminators' 1210 assert actual_resp_body == expected_resp_body 1211 conn.close() 1212 1213 1214class FaultySelect: 1215 """Mock class to insert errors in the selector.select method.""" 1216 1217 def __init__(self, original_select): 1218 """Initilize helper class to wrap the selector.select method.""" 1219 self.original_select = original_select 1220 self.request_served = False 1221 self.os_error_triggered = False 1222 1223 def __call__(self, timeout): 1224 """Intercept the calls to selector.select.""" 1225 if self.request_served: 1226 self.os_error_triggered = True 1227 raise OSError('Error while selecting the client socket.') 1228 1229 return self.original_select(timeout) 1230 1231 1232class FaultyGetMap: 1233 """Mock class to insert errors in the selector.get_map method.""" 1234 1235 def __init__(self, original_get_map): 1236 """Initilize helper class to wrap the selector.get_map method.""" 1237 self.original_get_map = original_get_map 1238 self.sabotage_conn = False 1239 self.conn_closed = False 1240 1241 def __call__(self): 1242 """Intercept the calls to selector.get_map.""" 1243 sabotage_targets = ( 1244 conn for _, (_, _, _, conn) in self.original_get_map().items() 1245 if isinstance(conn, cheroot.server.HTTPConnection) 1246 ) if self.sabotage_conn and not self.conn_closed else () 1247 1248 for conn in sabotage_targets: 1249 # close the socket to cause OSError 1250 conn.close() 1251 self.conn_closed = True 1252 1253 return self.original_get_map() 1254 1255 1256def test_invalid_selected_connection(test_client, monkeypatch): 1257 """Test the error handling segment of HTTP connection selection. 1258 1259 See :py:meth:`cheroot.connections.ConnectionManager.get_conn`. 1260 """ 1261 # patch the select method 1262 faux_select = FaultySelect( 1263 test_client.server_instance._connections._selector.select, 1264 ) 1265 monkeypatch.setattr( 1266 test_client.server_instance._connections._selector, 1267 'select', 1268 faux_select, 1269 ) 1270 1271 # patch the get_map method 1272 faux_get_map = FaultyGetMap( 1273 test_client.server_instance._connections._selector._selector.get_map, 1274 ) 1275 1276 monkeypatch.setattr( 1277 test_client.server_instance._connections._selector._selector, 1278 'get_map', 1279 faux_get_map, 1280 ) 1281 1282 # request a page with connection keep-alive to make sure 1283 # we'll have a connection to be modified. 1284 resp_status, resp_headers, resp_body = test_client.request( 1285 '/page1', headers=[('Connection', 'Keep-Alive')], 1286 ) 1287 1288 assert resp_status == '200 OK' 1289 # trigger the internal errors 1290 faux_get_map.sabotage_conn = faux_select.request_served = True 1291 # give time to make sure the error gets handled 1292 time.sleep(test_client.server_instance.expiration_interval * 2) 1293 assert faux_select.os_error_triggered 1294 assert faux_get_map.conn_closed 1295