1# Copyright (c) Twisted Matrix Laboratories. 2# See LICENSE for details. 3 4""" 5Tests for L{twisted.web._newclient}. 6""" 7 8__metaclass__ = type 9 10from zope.interface import implements 11from zope.interface.verify import verifyObject 12 13from twisted.python import log 14from twisted.python.failure import Failure 15from twisted.internet.interfaces import IConsumer, IPushProducer 16from twisted.internet.error import ConnectionDone, ConnectionLost 17from twisted.internet.defer import Deferred, succeed, fail, CancelledError 18from twisted.internet.protocol import Protocol 19from twisted.trial.unittest import TestCase 20from twisted.test.proto_helpers import StringTransport, AccumulatingProtocol 21from twisted.web._newclient import UNKNOWN_LENGTH, STATUS, HEADER, BODY, DONE 22from twisted.web._newclient import Request, Response, HTTPParser, HTTPClientParser 23from twisted.web._newclient import BadResponseVersion, ParseError, HTTP11ClientProtocol 24from twisted.web._newclient import ChunkedEncoder, RequestGenerationFailed 25from twisted.web._newclient import RequestTransmissionFailed, ResponseFailed 26from twisted.web._newclient import WrongBodyLength, RequestNotSent 27from twisted.web._newclient import ConnectionAborted, ResponseNeverReceived 28from twisted.web._newclient import BadHeaders, ResponseDone, PotentialDataLoss, ExcessWrite 29from twisted.web._newclient import TransportProxyProducer, LengthEnforcingConsumer, makeStatefulDispatcher 30from twisted.web.http_headers import Headers 31from twisted.web.http import _DataLoss 32from twisted.web.iweb import IBodyProducer, IResponse 33 34 35 36class StringTransport(StringTransport): 37 """ 38 A version of C{StringTransport} that supports C{abortConnection}. 39 """ 40 aborting = False 41 42 43 def abortConnection(self): 44 """ 45 A testable version of the C{ITCPTransport.abortConnection} method. 46 47 Since this is a special case of closing the connection, 48 C{loseConnection} is also called. 49 """ 50 self.aborting = True 51 self.loseConnection() 52 53 54 55class ArbitraryException(Exception): 56 """ 57 A unique, arbitrary exception type which L{twisted.web._newclient} knows 58 nothing about. 59 """ 60 61 62class AnotherArbitraryException(Exception): 63 """ 64 Similar to L{ArbitraryException} but with a different identity. 65 """ 66 67 68# A re-usable Headers instance for tests which don't really care what headers 69# they're sending. 70_boringHeaders = Headers({'host': ['example.com']}) 71 72 73def assertWrapperExceptionTypes(self, deferred, mainType, reasonTypes): 74 """ 75 Assert that the given L{Deferred} fails with the exception given by 76 C{mainType} and that the exceptions wrapped by the instance of C{mainType} 77 it fails with match the list of exception types given by C{reasonTypes}. 78 79 This is a helper for testing failures of exceptions which subclass 80 L{_newclient._WrapperException}. 81 82 @param self: A L{TestCase} instance which will be used to make the 83 assertions. 84 85 @param deferred: The L{Deferred} which is expected to fail with 86 C{mainType}. 87 88 @param mainType: A L{_newclient._WrapperException} subclass which will be 89 trapped on C{deferred}. 90 91 @param reasonTypes: A sequence of exception types which will be trapped on 92 the resulting C{mainType} exception instance's C{reasons} sequence. 93 94 @return: A L{Deferred} which fires with the C{mainType} instance 95 C{deferred} fails with, or which fails somehow. 96 """ 97 def cbFailed(err): 98 for reason, type in zip(err.reasons, reasonTypes): 99 reason.trap(type) 100 self.assertEqual(len(err.reasons), len(reasonTypes), 101 "len(%s) != len(%s)" % (err.reasons, reasonTypes)) 102 return err 103 d = self.assertFailure(deferred, mainType) 104 d.addCallback(cbFailed) 105 return d 106 107 108 109def assertResponseFailed(self, deferred, reasonTypes): 110 """ 111 A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType} 112 of L{ResponseFailed}. 113 """ 114 return assertWrapperExceptionTypes(self, deferred, ResponseFailed, reasonTypes) 115 116 117 118def assertRequestGenerationFailed(self, deferred, reasonTypes): 119 """ 120 A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType} 121 of L{RequestGenerationFailed}. 122 """ 123 return assertWrapperExceptionTypes(self, deferred, RequestGenerationFailed, reasonTypes) 124 125 126 127def assertRequestTransmissionFailed(self, deferred, reasonTypes): 128 """ 129 A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType} 130 of L{RequestTransmissionFailed}. 131 """ 132 return assertWrapperExceptionTypes(self, deferred, RequestTransmissionFailed, reasonTypes) 133 134 135 136def justTransportResponse(transport): 137 """ 138 Helper function for creating a Response which uses the given transport. 139 All of the other parameters to L{Response.__init__} are filled with 140 arbitrary values. Only use this method if you don't care about any of 141 them. 142 """ 143 return Response(('HTTP', 1, 1), 200, 'OK', _boringHeaders, transport) 144 145 146class MakeStatefulDispatcherTests(TestCase): 147 """ 148 Tests for L{makeStatefulDispatcher}. 149 """ 150 def test_functionCalledByState(self): 151 """ 152 A method defined with L{makeStatefulDispatcher} invokes a second 153 method based on the current state of the object. 154 """ 155 class Foo: 156 _state = 'A' 157 158 def bar(self): 159 pass 160 bar = makeStatefulDispatcher('quux', bar) 161 162 def _quux_A(self): 163 return 'a' 164 165 def _quux_B(self): 166 return 'b' 167 168 stateful = Foo() 169 self.assertEqual(stateful.bar(), 'a') 170 stateful._state = 'B' 171 self.assertEqual(stateful.bar(), 'b') 172 stateful._state = 'C' 173 self.assertRaises(RuntimeError, stateful.bar) 174 175 176 177class _HTTPParserTests(object): 178 """ 179 Base test class for L{HTTPParser} which is responsible for the bulk of 180 the task of parsing HTTP bytes. 181 """ 182 sep = None 183 184 def test_statusCallback(self): 185 """ 186 L{HTTPParser} calls its C{statusReceived} method when it receives a 187 status line. 188 """ 189 status = [] 190 protocol = HTTPParser() 191 protocol.statusReceived = status.append 192 protocol.makeConnection(StringTransport()) 193 self.assertEqual(protocol.state, STATUS) 194 protocol.dataReceived('HTTP/1.1 200 OK' + self.sep) 195 self.assertEqual(status, ['HTTP/1.1 200 OK']) 196 self.assertEqual(protocol.state, HEADER) 197 198 199 def _headerTestSetup(self): 200 header = {} 201 protocol = HTTPParser() 202 protocol.headerReceived = header.__setitem__ 203 protocol.makeConnection(StringTransport()) 204 protocol.dataReceived('HTTP/1.1 200 OK' + self.sep) 205 return header, protocol 206 207 208 def test_headerCallback(self): 209 """ 210 L{HTTPParser} calls its C{headerReceived} method when it receives a 211 header. 212 """ 213 header, protocol = self._headerTestSetup() 214 protocol.dataReceived('X-Foo:bar' + self.sep) 215 # Cannot tell it's not a continue header until the next line arrives 216 # and is not a continuation 217 protocol.dataReceived(self.sep) 218 self.assertEqual(header, {'X-Foo': 'bar'}) 219 self.assertEqual(protocol.state, BODY) 220 221 222 def test_continuedHeaderCallback(self): 223 """ 224 If a header is split over multiple lines, L{HTTPParser} calls 225 C{headerReceived} with the entire value once it is received. 226 """ 227 header, protocol = self._headerTestSetup() 228 protocol.dataReceived('X-Foo: bar' + self.sep) 229 protocol.dataReceived(' baz' + self.sep) 230 protocol.dataReceived('\tquux' + self.sep) 231 protocol.dataReceived(self.sep) 232 self.assertEqual(header, {'X-Foo': 'bar baz\tquux'}) 233 self.assertEqual(protocol.state, BODY) 234 235 236 def test_fieldContentWhitespace(self): 237 """ 238 Leading and trailing linear whitespace is stripped from the header 239 value passed to the C{headerReceived} callback. 240 """ 241 header, protocol = self._headerTestSetup() 242 value = ' \t %(sep)s bar \t%(sep)s \t%(sep)s' % dict(sep=self.sep) 243 protocol.dataReceived('X-Bar:' + value) 244 protocol.dataReceived('X-Foo:' + value) 245 protocol.dataReceived(self.sep) 246 self.assertEqual(header, {'X-Foo': 'bar', 247 'X-Bar': 'bar'}) 248 249 250 def test_allHeadersCallback(self): 251 """ 252 After the last header is received, L{HTTPParser} calls 253 C{allHeadersReceived}. 254 """ 255 called = [] 256 header, protocol = self._headerTestSetup() 257 def allHeadersReceived(): 258 called.append(protocol.state) 259 protocol.state = STATUS 260 protocol.allHeadersReceived = allHeadersReceived 261 protocol.dataReceived(self.sep) 262 self.assertEqual(called, [HEADER]) 263 self.assertEqual(protocol.state, STATUS) 264 265 266 def test_noHeaderCallback(self): 267 """ 268 If there are no headers in the message, L{HTTPParser} does not call 269 C{headerReceived}. 270 """ 271 header, protocol = self._headerTestSetup() 272 protocol.dataReceived(self.sep) 273 self.assertEqual(header, {}) 274 self.assertEqual(protocol.state, BODY) 275 276 277 def test_headersSavedOnResponse(self): 278 """ 279 All headers received by L{HTTPParser} are added to 280 L{HTTPParser.headers}. 281 """ 282 protocol = HTTPParser() 283 protocol.makeConnection(StringTransport()) 284 protocol.dataReceived('HTTP/1.1 200 OK' + self.sep) 285 protocol.dataReceived('X-Foo: bar' + self.sep) 286 protocol.dataReceived('X-Foo: baz' + self.sep) 287 protocol.dataReceived(self.sep) 288 expected = [('X-Foo', ['bar', 'baz'])] 289 self.assertEqual(expected, list(protocol.headers.getAllRawHeaders())) 290 291 292 def test_connectionControlHeaders(self): 293 """ 294 L{HTTPParser.isConnectionControlHeader} returns C{True} for headers 295 which are always connection control headers (similar to "hop-by-hop" 296 headers from RFC 2616 section 13.5.1) and C{False} for other headers. 297 """ 298 protocol = HTTPParser() 299 connHeaderNames = [ 300 'content-length', 'connection', 'keep-alive', 'te', 'trailers', 301 'transfer-encoding', 'upgrade', 'proxy-connection'] 302 303 for header in connHeaderNames: 304 self.assertTrue( 305 protocol.isConnectionControlHeader(header), 306 "Expecting %r to be a connection control header, but " 307 "wasn't" % (header,)) 308 self.assertFalse( 309 protocol.isConnectionControlHeader("date"), 310 "Expecting the arbitrarily selected 'date' header to not be " 311 "a connection control header, but was.") 312 313 314 def test_switchToBodyMode(self): 315 """ 316 L{HTTPParser.switchToBodyMode} raises L{RuntimeError} if called more 317 than once. 318 """ 319 protocol = HTTPParser() 320 protocol.makeConnection(StringTransport()) 321 protocol.switchToBodyMode(object()) 322 self.assertRaises(RuntimeError, protocol.switchToBodyMode, object()) 323 324 325 326class HTTPParserTestsRFCComplaintDelimeter(_HTTPParserTests, TestCase): 327 """ 328 L{_HTTPParserTests} using standard CR LF newlines. 329 """ 330 sep = '\r\n' 331 332 333 334class HTTPParserTestsNonRFCComplaintDelimeter(_HTTPParserTests, TestCase): 335 """ 336 L{_HTTPParserTests} using bare LF newlines. 337 """ 338 sep = '\n' 339 340 341 342class HTTPClientParserTests(TestCase): 343 """ 344 Tests for L{HTTPClientParser} which is responsible for parsing HTTP 345 response messages. 346 """ 347 def test_parseVersion(self): 348 """ 349 L{HTTPClientParser.parseVersion} parses a status line into its three 350 components. 351 """ 352 protocol = HTTPClientParser(None, None) 353 self.assertEqual( 354 protocol.parseVersion('CANDY/7.2'), 355 ('CANDY', 7, 2)) 356 357 358 def test_parseBadVersion(self): 359 """ 360 L{HTTPClientParser.parseVersion} raises L{ValueError} when passed an 361 unparsable version. 362 """ 363 protocol = HTTPClientParser(None, None) 364 e = BadResponseVersion 365 f = protocol.parseVersion 366 367 def checkParsing(s): 368 exc = self.assertRaises(e, f, s) 369 self.assertEqual(exc.data, s) 370 371 checkParsing('foo') 372 checkParsing('foo/bar/baz') 373 374 checkParsing('foo/') 375 checkParsing('foo/..') 376 377 checkParsing('foo/a.b') 378 checkParsing('foo/-1.-1') 379 380 381 def test_responseStatusParsing(self): 382 """ 383 L{HTTPClientParser.statusReceived} parses the version, code, and phrase 384 from the status line and stores them on the response object. 385 """ 386 request = Request('GET', '/', _boringHeaders, None) 387 protocol = HTTPClientParser(request, None) 388 protocol.makeConnection(StringTransport()) 389 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 390 self.assertEqual(protocol.response.version, ('HTTP', 1, 1)) 391 self.assertEqual(protocol.response.code, 200) 392 self.assertEqual(protocol.response.phrase, 'OK') 393 394 395 def test_badResponseStatus(self): 396 """ 397 L{HTTPClientParser.statusReceived} raises L{ParseError} if it is called 398 with a status line which cannot be parsed. 399 """ 400 protocol = HTTPClientParser(None, None) 401 402 def checkParsing(s): 403 exc = self.assertRaises(ParseError, protocol.statusReceived, s) 404 self.assertEqual(exc.data, s) 405 406 # If there are fewer than three whitespace-delimited parts to the 407 # status line, it is not valid and cannot be parsed. 408 checkParsing('foo') 409 checkParsing('HTTP/1.1 200') 410 411 # If the response code is not an integer, the status line is not valid 412 # and cannot be parsed. 413 checkParsing('HTTP/1.1 bar OK') 414 415 416 def _noBodyTest(self, request, status, response): 417 """ 418 Assert that L{HTTPClientParser} parses the given C{response} to 419 C{request}, resulting in a response with no body and no extra bytes and 420 leaving the transport in the producing state. 421 422 @param request: A L{Request} instance which might have caused a server 423 to return the given response. 424 @param status: A string giving the status line of the response to be 425 parsed. 426 @param response: A string giving the response to be parsed. 427 428 @return: A C{dict} of headers from the response. 429 """ 430 header = {} 431 finished = [] 432 body = [] 433 bodyDataFinished = [] 434 protocol = HTTPClientParser(request, finished.append) 435 protocol.headerReceived = header.__setitem__ 436 transport = StringTransport() 437 protocol.makeConnection(transport) 438 # Deliver just the status to initialize the response object so we can 439 # monkey-patch it to observe progress of the response parser. 440 protocol.dataReceived(status) 441 protocol.response._bodyDataReceived = body.append 442 protocol.response._bodyDataFinished = ( 443 lambda: bodyDataFinished.append(True)) 444 protocol.dataReceived(response) 445 self.assertEqual(transport.producerState, 'producing') 446 self.assertEqual(protocol.state, DONE) 447 self.assertEqual(body, []) 448 self.assertEqual(finished, ['']) 449 self.assertEqual(bodyDataFinished, [True]) 450 self.assertEqual(protocol.response.length, 0) 451 return header 452 453 454 def test_headResponse(self): 455 """ 456 If the response is to a HEAD request, no body is expected, the body 457 callback is not invoked, and the I{Content-Length} header is passed to 458 the header callback. 459 """ 460 request = Request('HEAD', '/', _boringHeaders, None) 461 status = 'HTTP/1.1 200 OK\r\n' 462 response = ( 463 'Content-Length: 10\r\n' 464 '\r\n') 465 header = self._noBodyTest(request, status, response) 466 self.assertEqual(header, {'Content-Length': '10'}) 467 468 469 def test_noContentResponse(self): 470 """ 471 If the response code is I{NO CONTENT} (204), no body is expected and 472 the body callback is not invoked. 473 """ 474 request = Request('GET', '/', _boringHeaders, None) 475 status = 'HTTP/1.1 204 NO CONTENT\r\n' 476 response = '\r\n' 477 self._noBodyTest(request, status, response) 478 479 480 def test_notModifiedResponse(self): 481 """ 482 If the response code is I{NOT MODIFIED} (304), no body is expected and 483 the body callback is not invoked. 484 """ 485 request = Request('GET', '/', _boringHeaders, None) 486 status = 'HTTP/1.1 304 NOT MODIFIED\r\n' 487 response = '\r\n' 488 self._noBodyTest(request, status, response) 489 490 491 def test_responseHeaders(self): 492 """ 493 The response headers are added to the response object's C{headers} 494 L{Headers} instance. 495 """ 496 protocol = HTTPClientParser( 497 Request('GET', '/', _boringHeaders, None), 498 lambda rest: None) 499 protocol.makeConnection(StringTransport()) 500 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 501 protocol.dataReceived('X-Foo: bar\r\n') 502 protocol.dataReceived('\r\n') 503 self.assertEqual( 504 protocol.connHeaders, 505 Headers({})) 506 self.assertEqual( 507 protocol.response.headers, 508 Headers({'x-foo': ['bar']})) 509 self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH) 510 511 512 def test_connectionHeaders(self): 513 """ 514 The connection control headers are added to the parser's C{connHeaders} 515 L{Headers} instance. 516 """ 517 protocol = HTTPClientParser( 518 Request('GET', '/', _boringHeaders, None), 519 lambda rest: None) 520 protocol.makeConnection(StringTransport()) 521 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 522 protocol.dataReceived('Content-Length: 123\r\n') 523 protocol.dataReceived('Connection: close\r\n') 524 protocol.dataReceived('\r\n') 525 self.assertEqual( 526 protocol.response.headers, 527 Headers({})) 528 self.assertEqual( 529 protocol.connHeaders, 530 Headers({'content-length': ['123'], 531 'connection': ['close']})) 532 self.assertEqual(protocol.response.length, 123) 533 534 535 def test_headResponseContentLengthEntityHeader(self): 536 """ 537 If a HEAD request is made, the I{Content-Length} header in the response 538 is added to the response headers, not the connection control headers. 539 """ 540 protocol = HTTPClientParser( 541 Request('HEAD', '/', _boringHeaders, None), 542 lambda rest: None) 543 protocol.makeConnection(StringTransport()) 544 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 545 protocol.dataReceived('Content-Length: 123\r\n') 546 protocol.dataReceived('\r\n') 547 self.assertEqual( 548 protocol.response.headers, 549 Headers({'content-length': ['123']})) 550 self.assertEqual( 551 protocol.connHeaders, 552 Headers({})) 553 self.assertEqual(protocol.response.length, 0) 554 555 556 def test_contentLength(self): 557 """ 558 If a response includes a body with a length given by the 559 I{Content-Length} header, the bytes which make up the body are passed 560 to the C{_bodyDataReceived} callback on the L{HTTPParser}. 561 """ 562 finished = [] 563 protocol = HTTPClientParser( 564 Request('GET', '/', _boringHeaders, None), 565 finished.append) 566 transport = StringTransport() 567 protocol.makeConnection(transport) 568 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 569 body = [] 570 protocol.response._bodyDataReceived = body.append 571 protocol.dataReceived('Content-Length: 10\r\n') 572 protocol.dataReceived('\r\n') 573 574 # Incidentally, the transport should be paused now. It is the response 575 # object's responsibility to resume this when it is ready for bytes. 576 self.assertEqual(transport.producerState, 'paused') 577 578 self.assertEqual(protocol.state, BODY) 579 protocol.dataReceived('x' * 6) 580 self.assertEqual(body, ['x' * 6]) 581 self.assertEqual(protocol.state, BODY) 582 protocol.dataReceived('y' * 4) 583 self.assertEqual(body, ['x' * 6, 'y' * 4]) 584 self.assertEqual(protocol.state, DONE) 585 self.assertTrue(finished, ['']) 586 587 588 def test_zeroContentLength(self): 589 """ 590 If a response includes a I{Content-Length} header indicating zero bytes 591 in the response, L{Response.length} is set accordingly and no data is 592 delivered to L{Response._bodyDataReceived}. 593 """ 594 finished = [] 595 protocol = HTTPClientParser( 596 Request('GET', '/', _boringHeaders, None), 597 finished.append) 598 599 protocol.makeConnection(StringTransport()) 600 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 601 602 body = [] 603 protocol.response._bodyDataReceived = body.append 604 605 protocol.dataReceived('Content-Length: 0\r\n') 606 protocol.dataReceived('\r\n') 607 608 self.assertEqual(protocol.state, DONE) 609 self.assertEqual(body, []) 610 self.assertTrue(finished, ['']) 611 self.assertEqual(protocol.response.length, 0) 612 613 614 615 def test_multipleContentLengthHeaders(self): 616 """ 617 If a response includes multiple I{Content-Length} headers, 618 L{HTTPClientParser.dataReceived} raises L{ValueError} to indicate that 619 the response is invalid and the transport is now unusable. 620 """ 621 protocol = HTTPClientParser( 622 Request('GET', '/', _boringHeaders, None), 623 None) 624 625 protocol.makeConnection(StringTransport()) 626 self.assertRaises( 627 ValueError, 628 protocol.dataReceived, 629 'HTTP/1.1 200 OK\r\n' 630 'Content-Length: 1\r\n' 631 'Content-Length: 2\r\n' 632 '\r\n') 633 634 635 def test_extraBytesPassedBack(self): 636 """ 637 If extra bytes are received past the end of a response, they are passed 638 to the finish callback. 639 """ 640 finished = [] 641 protocol = HTTPClientParser( 642 Request('GET', '/', _boringHeaders, None), 643 finished.append) 644 645 protocol.makeConnection(StringTransport()) 646 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 647 protocol.dataReceived('Content-Length: 0\r\n') 648 protocol.dataReceived('\r\nHere is another thing!') 649 self.assertEqual(protocol.state, DONE) 650 self.assertEqual(finished, ['Here is another thing!']) 651 652 653 def test_extraBytesPassedBackHEAD(self): 654 """ 655 If extra bytes are received past the end of the headers of a response 656 to a HEAD request, they are passed to the finish callback. 657 """ 658 finished = [] 659 protocol = HTTPClientParser( 660 Request('HEAD', '/', _boringHeaders, None), 661 finished.append) 662 663 protocol.makeConnection(StringTransport()) 664 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 665 protocol.dataReceived('Content-Length: 12\r\n') 666 protocol.dataReceived('\r\nHere is another thing!') 667 self.assertEqual(protocol.state, DONE) 668 self.assertEqual(finished, ['Here is another thing!']) 669 670 671 def test_chunkedResponseBody(self): 672 """ 673 If the response headers indicate the response body is encoded with the 674 I{chunked} transfer encoding, the body is decoded according to that 675 transfer encoding before being passed to L{Response._bodyDataReceived}. 676 """ 677 finished = [] 678 protocol = HTTPClientParser( 679 Request('GET', '/', _boringHeaders, None), 680 finished.append) 681 protocol.makeConnection(StringTransport()) 682 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 683 684 body = [] 685 protocol.response._bodyDataReceived = body.append 686 687 protocol.dataReceived('Transfer-Encoding: chunked\r\n') 688 protocol.dataReceived('\r\n') 689 690 # No data delivered yet 691 self.assertEqual(body, []) 692 693 # Cannot predict the length of a chunked encoded response body. 694 self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH) 695 696 # Deliver some chunks and make sure the data arrives 697 protocol.dataReceived('3\r\na') 698 self.assertEqual(body, ['a']) 699 protocol.dataReceived('bc\r\n') 700 self.assertEqual(body, ['a', 'bc']) 701 702 # The response's _bodyDataFinished method should be called when the last 703 # chunk is received. Extra data should be passed to the finished 704 # callback. 705 protocol.dataReceived('0\r\n\r\nextra') 706 self.assertEqual(finished, ['extra']) 707 708 709 def test_unknownContentLength(self): 710 """ 711 If a response does not include a I{Transfer-Encoding} or a 712 I{Content-Length}, the end of response body is indicated by the 713 connection being closed. 714 """ 715 finished = [] 716 protocol = HTTPClientParser( 717 Request('GET', '/', _boringHeaders, None), finished.append) 718 transport = StringTransport() 719 protocol.makeConnection(transport) 720 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 721 722 body = [] 723 protocol.response._bodyDataReceived = body.append 724 725 protocol.dataReceived('\r\n') 726 protocol.dataReceived('foo') 727 protocol.dataReceived('bar') 728 self.assertEqual(body, ['foo', 'bar']) 729 protocol.connectionLost(ConnectionDone("simulated end of connection")) 730 self.assertEqual(finished, ['']) 731 732 733 def test_contentLengthAndTransferEncoding(self): 734 """ 735 According to RFC 2616, section 4.4, point 3, if I{Content-Length} and 736 I{Transfer-Encoding: chunked} are present, I{Content-Length} MUST be 737 ignored 738 """ 739 finished = [] 740 protocol = HTTPClientParser( 741 Request('GET', '/', _boringHeaders, None), finished.append) 742 transport = StringTransport() 743 protocol.makeConnection(transport) 744 protocol.dataReceived('HTTP/1.1 200 OK\r\n') 745 746 body = [] 747 protocol.response._bodyDataReceived = body.append 748 749 protocol.dataReceived( 750 'Content-Length: 102\r\n' 751 'Transfer-Encoding: chunked\r\n' 752 '\r\n' 753 '3\r\n' 754 'abc\r\n' 755 '0\r\n' 756 '\r\n') 757 758 self.assertEqual(body, ['abc']) 759 self.assertEqual(finished, ['']) 760 761 762 def test_connectionLostBeforeBody(self): 763 """ 764 If L{HTTPClientParser.connectionLost} is called before the headers are 765 finished, the C{_responseDeferred} is fired with the L{Failure} passed 766 to C{connectionLost}. 767 """ 768 transport = StringTransport() 769 protocol = HTTPClientParser(Request('GET', '/', _boringHeaders, None), None) 770 protocol.makeConnection(transport) 771 # Grab this here because connectionLost gets rid of the attribute 772 responseDeferred = protocol._responseDeferred 773 protocol.connectionLost(Failure(ArbitraryException())) 774 775 return assertResponseFailed( 776 self, responseDeferred, [ArbitraryException]) 777 778 779 def test_connectionLostWithError(self): 780 """ 781 If one of the L{Response} methods called by 782 L{HTTPClientParser.connectionLost} raises an exception, the exception 783 is logged and not re-raised. 784 """ 785 transport = StringTransport() 786 protocol = HTTPClientParser(Request('GET', '/', _boringHeaders, None), 787 None) 788 protocol.makeConnection(transport) 789 790 response = [] 791 protocol._responseDeferred.addCallback(response.append) 792 protocol.dataReceived( 793 'HTTP/1.1 200 OK\r\n' 794 'Content-Length: 1\r\n' 795 '\r\n') 796 response = response[0] 797 798 # Arrange for an exception 799 def fakeBodyDataFinished(err=None): 800 raise ArbitraryException() 801 response._bodyDataFinished = fakeBodyDataFinished 802 803 protocol.connectionLost(None) 804 805 self.assertEqual(len(self.flushLoggedErrors(ArbitraryException)), 1) 806 807 808 def test_noResponseAtAll(self): 809 """ 810 If no response at all was received and the connection is lost, the 811 resulting error is L{ResponseNeverReceived}. 812 """ 813 protocol = HTTPClientParser( 814 Request('HEAD', '/', _boringHeaders, None), 815 lambda ign: None) 816 d = protocol._responseDeferred 817 818 protocol.makeConnection(StringTransport()) 819 protocol.connectionLost(ConnectionLost()) 820 return self.assertFailure(d, ResponseNeverReceived) 821 822 823 def test_someResponseButNotAll(self): 824 """ 825 If a partial response was received and the connection is lost, the 826 resulting error is L{ResponseFailed}, but not 827 L{ResponseNeverReceived}. 828 """ 829 protocol = HTTPClientParser( 830 Request('HEAD', '/', _boringHeaders, None), 831 lambda ign: None) 832 d = protocol._responseDeferred 833 834 protocol.makeConnection(StringTransport()) 835 protocol.dataReceived('2') 836 protocol.connectionLost(ConnectionLost()) 837 return self.assertFailure(d, ResponseFailed).addCallback( 838 self.assertIsInstance, ResponseFailed) 839 840 841 842class SlowRequest: 843 """ 844 L{SlowRequest} is a fake implementation of L{Request} which is easily 845 controlled externally (for example, by code in a test method). 846 847 @ivar stopped: A flag indicating whether C{stopWriting} has been called. 848 849 @ivar finished: After C{writeTo} is called, a L{Deferred} which was 850 returned by that method. L{SlowRequest} will never fire this 851 L{Deferred}. 852 """ 853 method = 'GET' 854 stopped = False 855 persistent = False 856 857 def writeTo(self, transport): 858 self.finished = Deferred() 859 return self.finished 860 861 862 def stopWriting(self): 863 self.stopped = True 864 865 866 867class SimpleRequest: 868 """ 869 L{SimpleRequest} is a fake implementation of L{Request} which writes a 870 short, fixed string to the transport passed to its C{writeTo} method and 871 returns a succeeded L{Deferred}. This vaguely emulates the behavior of a 872 L{Request} with no body producer. 873 """ 874 persistent = False 875 876 def writeTo(self, transport): 877 transport.write('SOME BYTES') 878 return succeed(None) 879 880 881 882class HTTP11ClientProtocolTests(TestCase): 883 """ 884 Tests for the HTTP 1.1 client protocol implementation, 885 L{HTTP11ClientProtocol}. 886 """ 887 def setUp(self): 888 """ 889 Create an L{HTTP11ClientProtocol} connected to a fake transport. 890 """ 891 self.transport = StringTransport() 892 self.protocol = HTTP11ClientProtocol() 893 self.protocol.makeConnection(self.transport) 894 895 896 def test_request(self): 897 """ 898 L{HTTP11ClientProtocol.request} accepts a L{Request} and calls its 899 C{writeTo} method with its own transport. 900 """ 901 self.protocol.request(SimpleRequest()) 902 self.assertEqual(self.transport.value(), 'SOME BYTES') 903 904 905 def test_secondRequest(self): 906 """ 907 The second time L{HTTP11ClientProtocol.request} is called, it returns a 908 L{Deferred} which immediately fires with a L{Failure} wrapping a 909 L{RequestNotSent} exception. 910 """ 911 self.protocol.request(SlowRequest()) 912 def cbNotSent(ignored): 913 self.assertEqual(self.transport.value(), '') 914 d = self.assertFailure( 915 self.protocol.request(SimpleRequest()), RequestNotSent) 916 d.addCallback(cbNotSent) 917 return d 918 919 920 def test_requestAfterConnectionLost(self): 921 """ 922 L{HTTP11ClientProtocol.request} returns a L{Deferred} which immediately 923 fires with a L{Failure} wrapping a L{RequestNotSent} if called after 924 the protocol has been disconnected. 925 """ 926 self.protocol.connectionLost( 927 Failure(ConnectionDone("sad transport"))) 928 def cbNotSent(ignored): 929 self.assertEqual(self.transport.value(), '') 930 d = self.assertFailure( 931 self.protocol.request(SimpleRequest()), RequestNotSent) 932 d.addCallback(cbNotSent) 933 return d 934 935 936 def test_failedWriteTo(self): 937 """ 938 If the L{Deferred} returned by L{Request.writeTo} fires with a 939 L{Failure}, L{HTTP11ClientProtocol.request} disconnects its transport 940 and returns a L{Deferred} which fires with a L{Failure} of 941 L{RequestGenerationFailed} wrapping the underlying failure. 942 """ 943 class BrokenRequest: 944 persistent = False 945 def writeTo(self, transport): 946 return fail(ArbitraryException()) 947 948 d = self.protocol.request(BrokenRequest()) 949 def cbFailed(ignored): 950 self.assertTrue(self.transport.disconnecting) 951 # Simulate what would happen if the protocol had a real transport 952 # and make sure no exception is raised. 953 self.protocol.connectionLost( 954 Failure(ConnectionDone("you asked for it"))) 955 d = assertRequestGenerationFailed(self, d, [ArbitraryException]) 956 d.addCallback(cbFailed) 957 return d 958 959 960 def test_synchronousWriteToError(self): 961 """ 962 If L{Request.writeTo} raises an exception, 963 L{HTTP11ClientProtocol.request} returns a L{Deferred} which fires with 964 a L{Failure} of L{RequestGenerationFailed} wrapping that exception. 965 """ 966 class BrokenRequest: 967 persistent = False 968 def writeTo(self, transport): 969 raise ArbitraryException() 970 971 d = self.protocol.request(BrokenRequest()) 972 return assertRequestGenerationFailed(self, d, [ArbitraryException]) 973 974 975 def test_connectionLostDuringRequestGeneration(self, mode=None): 976 """ 977 If L{HTTP11ClientProtocol}'s transport is disconnected before the 978 L{Deferred} returned by L{Request.writeTo} fires, the L{Deferred} 979 returned by L{HTTP11ClientProtocol.request} fires with a L{Failure} of 980 L{RequestTransmissionFailed} wrapping the underlying failure. 981 """ 982 request = SlowRequest() 983 d = self.protocol.request(request) 984 d = assertRequestTransmissionFailed(self, d, [ArbitraryException]) 985 986 # The connection hasn't been lost yet. The request should still be 987 # allowed to do its thing. 988 self.assertFalse(request.stopped) 989 990 self.protocol.connectionLost(Failure(ArbitraryException())) 991 992 # Now the connection has been lost. The request should have been told 993 # to stop writing itself. 994 self.assertTrue(request.stopped) 995 996 if mode == 'callback': 997 request.finished.callback(None) 998 elif mode == 'errback': 999 request.finished.errback(Failure(AnotherArbitraryException())) 1000 errors = self.flushLoggedErrors(AnotherArbitraryException) 1001 self.assertEqual(len(errors), 1) 1002 else: 1003 # Don't fire the writeTo Deferred at all. 1004 pass 1005 return d 1006 1007 1008 def test_connectionLostBeforeGenerationFinished(self): 1009 """ 1010 If the request passed to L{HTTP11ClientProtocol} finishes generation 1011 successfully after the L{HTTP11ClientProtocol}'s connection has been 1012 lost, nothing happens. 1013 """ 1014 return self.test_connectionLostDuringRequestGeneration('callback') 1015 1016 1017 def test_connectionLostBeforeGenerationFailed(self): 1018 """ 1019 If the request passed to L{HTTP11ClientProtocol} finished generation 1020 with an error after the L{HTTP11ClientProtocol}'s connection has been 1021 lost, nothing happens. 1022 """ 1023 return self.test_connectionLostDuringRequestGeneration('errback') 1024 1025 1026 def test_errorMessageOnConnectionLostBeforeGenerationFailedDoesNotConfuse(self): 1027 """ 1028 If the request passed to L{HTTP11ClientProtocol} finished generation 1029 with an error after the L{HTTP11ClientProtocol}'s connection has been 1030 lost, an error is logged that gives a non-confusing hint to user on what 1031 went wrong. 1032 """ 1033 errors = [] 1034 log.addObserver(errors.append) 1035 self.addCleanup(log.removeObserver, errors.append) 1036 1037 def check(ignore): 1038 error = errors[0] 1039 self.assertEqual(error['why'], 1040 'Error writing request, but not in valid state ' 1041 'to finalize request: CONNECTION_LOST') 1042 1043 return self.test_connectionLostDuringRequestGeneration( 1044 'errback').addCallback(check) 1045 1046 1047 def test_receiveSimplestResponse(self): 1048 """ 1049 When a response is delivered to L{HTTP11ClientProtocol}, the 1050 L{Deferred} previously returned by the C{request} method is called back 1051 with a L{Response} instance and the connection is closed. 1052 """ 1053 d = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1054 def cbRequest(response): 1055 self.assertEqual(response.code, 200) 1056 self.assertEqual(response.headers, Headers()) 1057 self.assertTrue(self.transport.disconnecting) 1058 self.assertEqual(self.protocol.state, 'QUIESCENT') 1059 d.addCallback(cbRequest) 1060 self.protocol.dataReceived( 1061 "HTTP/1.1 200 OK\r\n" 1062 "Content-Length: 0\r\n" 1063 "Connection: close\r\n" 1064 "\r\n") 1065 return d 1066 1067 1068 def test_receiveResponseHeaders(self): 1069 """ 1070 The headers included in a response delivered to L{HTTP11ClientProtocol} 1071 are included on the L{Response} instance passed to the callback 1072 returned by the C{request} method. 1073 """ 1074 d = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1075 def cbRequest(response): 1076 expected = Headers({'x-foo': ['bar', 'baz']}) 1077 self.assertEqual(response.headers, expected) 1078 d.addCallback(cbRequest) 1079 self.protocol.dataReceived( 1080 "HTTP/1.1 200 OK\r\n" 1081 "X-Foo: bar\r\n" 1082 "X-Foo: baz\r\n" 1083 "\r\n") 1084 return d 1085 1086 1087 def test_receiveResponseBeforeRequestGenerationDone(self): 1088 """ 1089 If response bytes are delivered to L{HTTP11ClientProtocol} before the 1090 L{Deferred} returned by L{Request.writeTo} fires, those response bytes 1091 are parsed as part of the response. 1092 1093 The connection is also closed, because we're in a confusing state, and 1094 therefore the C{quiescentCallback} isn't called. 1095 """ 1096 quiescentResult = [] 1097 transport = StringTransport() 1098 protocol = HTTP11ClientProtocol(quiescentResult.append) 1099 protocol.makeConnection(transport) 1100 1101 request = SlowRequest() 1102 d = protocol.request(request) 1103 protocol.dataReceived( 1104 "HTTP/1.1 200 OK\r\n" 1105 "X-Foo: bar\r\n" 1106 "Content-Length: 6\r\n" 1107 "\r\n" 1108 "foobar") 1109 def cbResponse(response): 1110 p = AccumulatingProtocol() 1111 whenFinished = p.closedDeferred = Deferred() 1112 response.deliverBody(p) 1113 self.assertEqual( 1114 protocol.state, 'TRANSMITTING_AFTER_RECEIVING_RESPONSE') 1115 self.assertTrue(transport.disconnecting) 1116 self.assertEqual(quiescentResult, []) 1117 return whenFinished.addCallback( 1118 lambda ign: (response, p.data)) 1119 d.addCallback(cbResponse) 1120 def cbAllResponse((response, body)): 1121 self.assertEqual(response.version, ('HTTP', 1, 1)) 1122 self.assertEqual(response.code, 200) 1123 self.assertEqual(response.phrase, 'OK') 1124 self.assertEqual(response.headers, Headers({'x-foo': ['bar']})) 1125 self.assertEqual(body, "foobar") 1126 1127 # Also nothing bad should happen if the request does finally 1128 # finish, even though it is completely irrelevant. 1129 request.finished.callback(None) 1130 1131 d.addCallback(cbAllResponse) 1132 return d 1133 1134 1135 def test_connectionLostAfterReceivingResponseBeforeRequestGenerationDone(self): 1136 """ 1137 If response bytes are delivered to L{HTTP11ClientProtocol} before the 1138 request completes, calling C{connectionLost} on the protocol will 1139 result in protocol being moved to C{'CONNECTION_LOST'} state. 1140 """ 1141 request = SlowRequest() 1142 d = self.protocol.request(request) 1143 self.protocol.dataReceived( 1144 "HTTP/1.1 400 BAD REQUEST\r\n" 1145 "Content-Length: 9\r\n" 1146 "\r\n" 1147 "tisk tisk") 1148 def cbResponse(response): 1149 p = AccumulatingProtocol() 1150 whenFinished = p.closedDeferred = Deferred() 1151 response.deliverBody(p) 1152 return whenFinished.addCallback( 1153 lambda ign: (response, p.data)) 1154 d.addCallback(cbResponse) 1155 def cbAllResponse(ignore): 1156 request.finished.callback(None) 1157 # Nothing dire will happen when the connection is lost 1158 self.protocol.connectionLost(Failure(ArbitraryException())) 1159 self.assertEqual(self.protocol._state, 'CONNECTION_LOST') 1160 d.addCallback(cbAllResponse) 1161 return d 1162 1163 1164 def test_receiveResponseBody(self): 1165 """ 1166 The C{deliverBody} method of the response object with which the 1167 L{Deferred} returned by L{HTTP11ClientProtocol.request} fires can be 1168 used to get the body of the response. 1169 """ 1170 protocol = AccumulatingProtocol() 1171 whenFinished = protocol.closedDeferred = Deferred() 1172 requestDeferred = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1173 1174 self.protocol.dataReceived( 1175 "HTTP/1.1 200 OK\r\n" 1176 "Content-Length: 6\r\n" 1177 "\r") 1178 1179 # Here's what's going on: all the response headers have been delivered 1180 # by this point, so the request Deferred can fire with a Response 1181 # object. The body is yet to come, but that's okay, because the 1182 # Response object is how you *get* the body. 1183 result = [] 1184 requestDeferred.addCallback(result.append) 1185 1186 self.assertEqual(result, []) 1187 # Deliver the very last byte of the response. It is exactly at this 1188 # point which the Deferred returned by request should fire. 1189 self.protocol.dataReceived("\n") 1190 response = result[0] 1191 1192 response.deliverBody(protocol) 1193 1194 self.protocol.dataReceived("foo") 1195 self.protocol.dataReceived("bar") 1196 1197 def cbAllResponse(ignored): 1198 self.assertEqual(protocol.data, "foobar") 1199 protocol.closedReason.trap(ResponseDone) 1200 whenFinished.addCallback(cbAllResponse) 1201 return whenFinished 1202 1203 1204 def test_responseBodyFinishedWhenConnectionLostWhenContentLengthIsUnknown( 1205 self): 1206 """ 1207 If the length of the response body is unknown, the protocol passed to 1208 the response's C{deliverBody} method has its C{connectionLost} 1209 method called with a L{Failure} wrapping a L{PotentialDataLoss} 1210 exception. 1211 """ 1212 requestDeferred = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1213 self.protocol.dataReceived( 1214 "HTTP/1.1 200 OK\r\n" 1215 "\r\n") 1216 1217 result = [] 1218 requestDeferred.addCallback(result.append) 1219 response = result[0] 1220 1221 protocol = AccumulatingProtocol() 1222 response.deliverBody(protocol) 1223 1224 self.protocol.dataReceived("foo") 1225 self.protocol.dataReceived("bar") 1226 1227 self.assertEqual(protocol.data, "foobar") 1228 self.protocol.connectionLost( 1229 Failure(ConnectionDone("low-level transport disconnected"))) 1230 1231 protocol.closedReason.trap(PotentialDataLoss) 1232 1233 1234 def test_chunkedResponseBodyUnfinishedWhenConnectionLost(self): 1235 """ 1236 If the final chunk has not been received when the connection is lost 1237 (for any reason), the protocol passed to C{deliverBody} has its 1238 C{connectionLost} method called with a L{Failure} wrapping the 1239 exception for that reason. 1240 """ 1241 requestDeferred = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1242 self.protocol.dataReceived( 1243 "HTTP/1.1 200 OK\r\n" 1244 "Transfer-Encoding: chunked\r\n" 1245 "\r\n") 1246 1247 result = [] 1248 requestDeferred.addCallback(result.append) 1249 response = result[0] 1250 1251 protocol = AccumulatingProtocol() 1252 response.deliverBody(protocol) 1253 1254 self.protocol.dataReceived("3\r\nfoo\r\n") 1255 self.protocol.dataReceived("3\r\nbar\r\n") 1256 1257 self.assertEqual(protocol.data, "foobar") 1258 1259 self.protocol.connectionLost(Failure(ArbitraryException())) 1260 1261 return assertResponseFailed( 1262 self, fail(protocol.closedReason), [ArbitraryException, _DataLoss]) 1263 1264 1265 def test_parserDataReceivedException(self): 1266 """ 1267 If the parser L{HTTP11ClientProtocol} delivers bytes to in 1268 C{dataReceived} raises an exception, the exception is wrapped in a 1269 L{Failure} and passed to the parser's C{connectionLost} and then the 1270 L{HTTP11ClientProtocol}'s transport is disconnected. 1271 """ 1272 requestDeferred = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1273 self.protocol.dataReceived('unparseable garbage goes here\r\n') 1274 d = assertResponseFailed(self, requestDeferred, [ParseError]) 1275 def cbFailed(exc): 1276 self.assertTrue(self.transport.disconnecting) 1277 self.assertEqual( 1278 exc.reasons[0].value.data, 'unparseable garbage goes here') 1279 1280 # Now do what StringTransport doesn't do but a real transport would 1281 # have, call connectionLost on the HTTP11ClientProtocol. Nothing 1282 # is asserted about this, but it's important for it to not raise an 1283 # exception. 1284 self.protocol.connectionLost(Failure(ConnectionDone("it is done"))) 1285 1286 d.addCallback(cbFailed) 1287 return d 1288 1289 1290 def test_proxyStopped(self): 1291 """ 1292 When the HTTP response parser is disconnected, the 1293 L{TransportProxyProducer} which was connected to it as a transport is 1294 stopped. 1295 """ 1296 requestDeferred = self.protocol.request(Request('GET', '/', _boringHeaders, None)) 1297 transport = self.protocol._parser.transport 1298 self.assertIdentical(transport._producer, self.transport) 1299 self.protocol._disconnectParser(Failure(ConnectionDone("connection done"))) 1300 self.assertIdentical(transport._producer, None) 1301 return assertResponseFailed(self, requestDeferred, [ConnectionDone]) 1302 1303 1304 def test_abortClosesConnection(self): 1305 """ 1306 L{HTTP11ClientProtocol.abort} will tell the transport to close its 1307 connection when it is invoked, and returns a C{Deferred} that fires 1308 when the connection is lost. 1309 """ 1310 transport = StringTransport() 1311 protocol = HTTP11ClientProtocol() 1312 protocol.makeConnection(transport) 1313 r1 = [] 1314 r2 = [] 1315 protocol.abort().addCallback(r1.append) 1316 protocol.abort().addCallback(r2.append) 1317 self.assertEqual((r1, r2), ([], [])) 1318 self.assertTrue(transport.disconnecting) 1319 1320 # Disconnect protocol, the Deferreds will fire: 1321 protocol.connectionLost(Failure(ConnectionDone())) 1322 self.assertEqual(r1, [None]) 1323 self.assertEqual(r2, [None]) 1324 1325 1326 def test_abortAfterConnectionLost(self): 1327 """ 1328 L{HTTP11ClientProtocol.abort} called after the connection is lost 1329 returns a C{Deferred} that fires immediately. 1330 """ 1331 transport = StringTransport() 1332 protocol = HTTP11ClientProtocol() 1333 protocol.makeConnection(transport) 1334 protocol.connectionLost(Failure(ConnectionDone())) 1335 1336 result = [] 1337 protocol.abort().addCallback(result.append) 1338 self.assertEqual(result, [None]) 1339 self.assertEqual(protocol._state, "CONNECTION_LOST") 1340 1341 1342 def test_abortBeforeResponseBody(self): 1343 """ 1344 The Deferred returned by L{HTTP11ClientProtocol.request} will fire 1345 with a L{ResponseFailed} failure containing a L{ConnectionAborted} 1346 exception, if the connection was aborted before all response headers 1347 have been received. 1348 """ 1349 transport = StringTransport() 1350 protocol = HTTP11ClientProtocol() 1351 protocol.makeConnection(transport) 1352 result = protocol.request(Request('GET', '/', _boringHeaders, None)) 1353 protocol.abort() 1354 self.assertTrue(transport.disconnecting) 1355 protocol.connectionLost(Failure(ConnectionDone())) 1356 return assertResponseFailed(self, result, [ConnectionAborted]) 1357 1358 1359 def test_abortAfterResponseHeaders(self): 1360 """ 1361 When the connection is aborted after the response headers have 1362 been received and the L{Response} has been made available to 1363 application code, the response body protocol's C{connectionLost} 1364 method will be invoked with a L{ResponseFailed} failure containing a 1365 L{ConnectionAborted} exception. 1366 """ 1367 transport = StringTransport() 1368 protocol = HTTP11ClientProtocol() 1369 protocol.makeConnection(transport) 1370 result = protocol.request(Request('GET', '/', _boringHeaders, None)) 1371 1372 protocol.dataReceived( 1373 "HTTP/1.1 200 OK\r\n" 1374 "Content-Length: 1\r\n" 1375 "\r\n" 1376 ) 1377 1378 testResult = Deferred() 1379 1380 class BodyDestination(Protocol): 1381 """ 1382 A body response protocol which immediately aborts the HTTP 1383 connection. 1384 """ 1385 def connectionMade(self): 1386 """ 1387 Abort the HTTP connection. 1388 """ 1389 protocol.abort() 1390 1391 def connectionLost(self, reason): 1392 """ 1393 Make the reason for the losing of the connection available to 1394 the unit test via C{testResult}. 1395 """ 1396 testResult.errback(reason) 1397 1398 1399 def deliverBody(response): 1400 """ 1401 Connect the L{BodyDestination} response body protocol to the 1402 response, and then simulate connection loss after ensuring that 1403 the HTTP connection has been aborted. 1404 """ 1405 response.deliverBody(BodyDestination()) 1406 self.assertTrue(transport.disconnecting) 1407 protocol.connectionLost(Failure(ConnectionDone())) 1408 1409 1410 def checkError(error): 1411 self.assertIsInstance(error.response, Response) 1412 1413 1414 result.addCallback(deliverBody) 1415 deferred = assertResponseFailed(self, testResult, 1416 [ConnectionAborted, _DataLoss]) 1417 return deferred.addCallback(checkError) 1418 1419 1420 def test_quiescentCallbackCalled(self): 1421 """ 1422 If after a response is done the {HTTP11ClientProtocol} stays open and 1423 returns to QUIESCENT state, all per-request state is reset and the 1424 C{quiescentCallback} is called with the protocol instance. 1425 1426 This is useful for implementing a persistent connection pool. 1427 1428 The C{quiescentCallback} is called *before* the response-receiving 1429 protocol's C{connectionLost}, so that new requests triggered by end of 1430 first request can re-use a persistent connection. 1431 """ 1432 quiescentResult = [] 1433 def callback(p): 1434 self.assertEqual(p, protocol) 1435 self.assertEqual(p.state, "QUIESCENT") 1436 quiescentResult.append(p) 1437 1438 transport = StringTransport() 1439 protocol = HTTP11ClientProtocol(callback) 1440 protocol.makeConnection(transport) 1441 1442 requestDeferred = protocol.request( 1443 Request('GET', '/', _boringHeaders, None, persistent=True)) 1444 protocol.dataReceived( 1445 "HTTP/1.1 200 OK\r\n" 1446 "Content-length: 3\r\n" 1447 "\r\n") 1448 1449 # Headers done, but still no quiescent callback: 1450 self.assertEqual(quiescentResult, []) 1451 1452 result = [] 1453 requestDeferred.addCallback(result.append) 1454 response = result[0] 1455 1456 # When response body is done (i.e. connectionLost is called), note the 1457 # fact in quiescentResult: 1458 bodyProtocol = AccumulatingProtocol() 1459 bodyProtocol.closedDeferred = Deferred() 1460 bodyProtocol.closedDeferred.addCallback( 1461 lambda ign: quiescentResult.append("response done")) 1462 1463 response.deliverBody(bodyProtocol) 1464 protocol.dataReceived("abc") 1465 bodyProtocol.closedReason.trap(ResponseDone) 1466 # Quiescent callback called *before* protocol handling the response 1467 # body gets its connectionLost called: 1468 self.assertEqual(quiescentResult, [protocol, "response done"]) 1469 1470 # Make sure everything was cleaned up: 1471 self.assertEqual(protocol._parser, None) 1472 self.assertEqual(protocol._finishedRequest, None) 1473 self.assertEqual(protocol._currentRequest, None) 1474 self.assertEqual(protocol._transportProxy, None) 1475 self.assertEqual(protocol._responseDeferred, None) 1476 1477 1478 def test_transportProducingWhenQuiescentAfterFullBody(self): 1479 """ 1480 The C{quiescentCallback} passed to L{HTTP11ClientProtocol} will only be 1481 invoked once that protocol is in a state similar to its initial state. 1482 One of the aspects of this initial state is the producer-state of its 1483 transport; an L{HTTP11ClientProtocol} begins with a transport that is 1484 producing, i.e. not C{pauseProducing}'d. 1485 1486 Therefore, when C{quiescentCallback} is invoked the protocol will still 1487 be producing. 1488 """ 1489 quiescentResult = [] 1490 def callback(p): 1491 self.assertEqual(p, protocol) 1492 self.assertEqual(p.state, "QUIESCENT") 1493 quiescentResult.append(p) 1494 1495 transport = StringTransport() 1496 protocol = HTTP11ClientProtocol(callback) 1497 protocol.makeConnection(transport) 1498 requestDeferred = protocol.request( 1499 Request('GET', '/', _boringHeaders, None, persistent=True)) 1500 protocol.dataReceived( 1501 "HTTP/1.1 200 OK\r\n" 1502 "Content-length: 3\r\n" 1503 "\r\n" 1504 "BBB" # _full_ content of the response. 1505 ) 1506 1507 response = self.successResultOf(requestDeferred) 1508 # Sanity check: response should have full response body, just waiting 1509 # for deliverBody 1510 self.assertEqual(response._state, 'DEFERRED_CLOSE') 1511 1512 # The transport is quiescent, because the response has been received. 1513 # If we were connection pooling here, it would have been returned to 1514 # the pool. 1515 self.assertEqual(len(quiescentResult), 1) 1516 1517 # And that transport is totally still reading, right? Because it would 1518 # leak forever if it were sitting there disconnected from the 1519 # reactor... 1520 self.assertEqual(transport.producerState, 'producing') 1521 1522 1523 def test_quiescentCallbackCalledEmptyResponse(self): 1524 """ 1525 The quiescentCallback is called before the request C{Deferred} fires, 1526 in cases where the response has no body. 1527 """ 1528 quiescentResult = [] 1529 def callback(p): 1530 self.assertEqual(p, protocol) 1531 self.assertEqual(p.state, "QUIESCENT") 1532 quiescentResult.append(p) 1533 1534 transport = StringTransport() 1535 protocol = HTTP11ClientProtocol(callback) 1536 protocol.makeConnection(transport) 1537 1538 requestDeferred = protocol.request( 1539 Request('GET', '/', _boringHeaders, None, persistent=True)) 1540 requestDeferred.addCallback(quiescentResult.append) 1541 protocol.dataReceived( 1542 "HTTP/1.1 200 OK\r\n" 1543 "Content-length: 0\r\n" 1544 "\r\n") 1545 1546 self.assertEqual(len(quiescentResult), 2) 1547 self.assertIdentical(quiescentResult[0], protocol) 1548 self.assertIsInstance(quiescentResult[1], Response) 1549 1550 1551 def test_quiescentCallbackNotCalled(self): 1552 """ 1553 If after a response is done the {HTTP11ClientProtocol} returns a 1554 C{Connection: close} header in the response, the C{quiescentCallback} 1555 is not called and the connection is lost. 1556 """ 1557 quiescentResult = [] 1558 transport = StringTransport() 1559 protocol = HTTP11ClientProtocol(quiescentResult.append) 1560 protocol.makeConnection(transport) 1561 1562 requestDeferred = protocol.request( 1563 Request('GET', '/', _boringHeaders, None, persistent=True)) 1564 protocol.dataReceived( 1565 "HTTP/1.1 200 OK\r\n" 1566 "Content-length: 0\r\n" 1567 "Connection: close\r\n" 1568 "\r\n") 1569 1570 result = [] 1571 requestDeferred.addCallback(result.append) 1572 response = result[0] 1573 1574 bodyProtocol = AccumulatingProtocol() 1575 response.deliverBody(bodyProtocol) 1576 bodyProtocol.closedReason.trap(ResponseDone) 1577 self.assertEqual(quiescentResult, []) 1578 self.assertTrue(transport.disconnecting) 1579 1580 1581 def test_quiescentCallbackNotCalledNonPersistentQuery(self): 1582 """ 1583 If the request was non-persistent (i.e. sent C{Connection: close}), 1584 the C{quiescentCallback} is not called and the connection is lost. 1585 """ 1586 quiescentResult = [] 1587 transport = StringTransport() 1588 protocol = HTTP11ClientProtocol(quiescentResult.append) 1589 protocol.makeConnection(transport) 1590 1591 requestDeferred = protocol.request( 1592 Request('GET', '/', _boringHeaders, None, persistent=False)) 1593 protocol.dataReceived( 1594 "HTTP/1.1 200 OK\r\n" 1595 "Content-length: 0\r\n" 1596 "\r\n") 1597 1598 result = [] 1599 requestDeferred.addCallback(result.append) 1600 response = result[0] 1601 1602 bodyProtocol = AccumulatingProtocol() 1603 response.deliverBody(bodyProtocol) 1604 bodyProtocol.closedReason.trap(ResponseDone) 1605 self.assertEqual(quiescentResult, []) 1606 self.assertTrue(transport.disconnecting) 1607 1608 1609 def test_quiescentCallbackThrows(self): 1610 """ 1611 If C{quiescentCallback} throws an exception, the error is logged and 1612 protocol is disconnected. 1613 """ 1614 def callback(p): 1615 raise ZeroDivisionError() 1616 1617 transport = StringTransport() 1618 protocol = HTTP11ClientProtocol(callback) 1619 protocol.makeConnection(transport) 1620 1621 requestDeferred = protocol.request( 1622 Request('GET', '/', _boringHeaders, None, persistent=True)) 1623 protocol.dataReceived( 1624 "HTTP/1.1 200 OK\r\n" 1625 "Content-length: 0\r\n" 1626 "\r\n") 1627 1628 result = [] 1629 requestDeferred.addCallback(result.append) 1630 response = result[0] 1631 bodyProtocol = AccumulatingProtocol() 1632 response.deliverBody(bodyProtocol) 1633 bodyProtocol.closedReason.trap(ResponseDone) 1634 1635 errors = self.flushLoggedErrors(ZeroDivisionError) 1636 self.assertEqual(len(errors), 1) 1637 self.assertTrue(transport.disconnecting) 1638 1639 1640 def test_cancelBeforeResponse(self): 1641 """ 1642 The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire 1643 with a L{ResponseNeverReceived} failure containing a L{CancelledError} 1644 exception if the request was cancelled before any response headers were 1645 received. 1646 """ 1647 transport = StringTransport() 1648 protocol = HTTP11ClientProtocol() 1649 protocol.makeConnection(transport) 1650 result = protocol.request(Request('GET', '/', _boringHeaders, None)) 1651 result.cancel() 1652 self.assertTrue(transport.aborting) 1653 return assertWrapperExceptionTypes( 1654 self, result, ResponseNeverReceived, [CancelledError]) 1655 1656 1657 def test_cancelDuringResponse(self): 1658 """ 1659 The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire 1660 with a L{ResponseFailed} failure containing a L{CancelledError} 1661 exception if the request was cancelled before all response headers were 1662 received. 1663 """ 1664 transport = StringTransport() 1665 protocol = HTTP11ClientProtocol() 1666 protocol.makeConnection(transport) 1667 result = protocol.request(Request('GET', '/', _boringHeaders, None)) 1668 protocol.dataReceived("HTTP/1.1 200 OK\r\n") 1669 result.cancel() 1670 self.assertTrue(transport.aborting) 1671 return assertResponseFailed(self, result, [CancelledError]) 1672 1673 1674 def assertCancelDuringBodyProduction(self, producerLength): 1675 """ 1676 The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire 1677 with a L{RequestGenerationFailed} failure containing a 1678 L{CancelledError} exception if the request was cancelled before a 1679 C{bodyProducer} has finished producing. 1680 """ 1681 transport = StringTransport() 1682 protocol = HTTP11ClientProtocol() 1683 protocol.makeConnection(transport) 1684 producer = StringProducer(producerLength) 1685 1686 nonlocal = {'cancelled': False} 1687 def cancel(ign): 1688 nonlocal['cancelled'] = True 1689 def startProducing(consumer): 1690 producer.consumer = consumer 1691 producer.finished = Deferred(cancel) 1692 return producer.finished 1693 producer.startProducing = startProducing 1694 1695 result = protocol.request(Request('POST', '/bar', _boringHeaders, producer)) 1696 producer.consumer.write('x' * 5) 1697 result.cancel() 1698 self.assertTrue(transport.aborting) 1699 self.assertTrue(nonlocal['cancelled']) 1700 return assertRequestGenerationFailed(self, result, [CancelledError]) 1701 1702 1703 def test_cancelDuringBodyProduction(self): 1704 """ 1705 The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire 1706 with a L{RequestGenerationFailed} failure containing a 1707 L{CancelledError} exception if the request was cancelled before a 1708 C{bodyProducer} with an explicit length has finished producing. 1709 """ 1710 return self.assertCancelDuringBodyProduction(10) 1711 1712 1713 def test_cancelDuringChunkedBodyProduction(self): 1714 """ 1715 The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire 1716 with a L{RequestGenerationFailed} failure containing a 1717 L{CancelledError} exception if the request was cancelled before a 1718 C{bodyProducer} with C{UNKNOWN_LENGTH} has finished producing. 1719 """ 1720 return self.assertCancelDuringBodyProduction(UNKNOWN_LENGTH) 1721 1722 1723 1724class StringProducer: 1725 """ 1726 L{StringProducer} is a dummy body producer. 1727 1728 @ivar stopped: A flag which indicates whether or not C{stopProducing} has 1729 been called. 1730 @ivar consumer: After C{startProducing} is called, the value of the 1731 C{consumer} argument to that method. 1732 @ivar finished: After C{startProducing} is called, a L{Deferred} which was 1733 returned by that method. L{StringProducer} will never fire this 1734 L{Deferred}. 1735 """ 1736 implements(IBodyProducer) 1737 1738 stopped = False 1739 1740 def __init__(self, length): 1741 self.length = length 1742 1743 1744 def startProducing(self, consumer): 1745 self.consumer = consumer 1746 self.finished = Deferred() 1747 return self.finished 1748 1749 1750 def stopProducing(self): 1751 self.stopped = True 1752 1753 1754 1755class RequestTests(TestCase): 1756 """ 1757 Tests for L{Request}. 1758 """ 1759 def setUp(self): 1760 self.transport = StringTransport() 1761 1762 1763 def test_sendSimplestRequest(self): 1764 """ 1765 L{Request.writeTo} formats the request data and writes it to the given 1766 transport. 1767 """ 1768 Request('GET', '/', _boringHeaders, None).writeTo(self.transport) 1769 self.assertEqual( 1770 self.transport.value(), 1771 "GET / HTTP/1.1\r\n" 1772 "Connection: close\r\n" 1773 "Host: example.com\r\n" 1774 "\r\n") 1775 1776 1777 def test_sendSimplestPersistentRequest(self): 1778 """ 1779 A pesistent request does not send 'Connection: close' header. 1780 """ 1781 req = Request('GET', '/', _boringHeaders, None, persistent=True) 1782 req.writeTo(self.transport) 1783 self.assertEqual( 1784 self.transport.value(), 1785 "GET / HTTP/1.1\r\n" 1786 "Host: example.com\r\n" 1787 "\r\n") 1788 1789 1790 def test_sendRequestHeaders(self): 1791 """ 1792 L{Request.writeTo} formats header data and writes it to the given 1793 transport. 1794 """ 1795 headers = Headers({'x-foo': ['bar', 'baz'], 'host': ['example.com']}) 1796 Request('GET', '/foo', headers, None).writeTo(self.transport) 1797 lines = self.transport.value().split('\r\n') 1798 self.assertEqual(lines[0], "GET /foo HTTP/1.1") 1799 self.assertEqual(lines[-2:], ["", ""]) 1800 del lines[0], lines[-2:] 1801 lines.sort() 1802 self.assertEqual( 1803 lines, 1804 ["Connection: close", 1805 "Host: example.com", 1806 "X-Foo: bar", 1807 "X-Foo: baz"]) 1808 1809 1810 def test_sendChunkedRequestBody(self): 1811 """ 1812 L{Request.writeTo} uses chunked encoding to write data from the request 1813 body producer to the given transport. It registers the request body 1814 producer with the transport. 1815 """ 1816 producer = StringProducer(UNKNOWN_LENGTH) 1817 request = Request('POST', '/bar', _boringHeaders, producer) 1818 request.writeTo(self.transport) 1819 1820 self.assertNotIdentical(producer.consumer, None) 1821 self.assertIdentical(self.transport.producer, producer) 1822 self.assertTrue(self.transport.streaming) 1823 1824 self.assertEqual( 1825 self.transport.value(), 1826 "POST /bar HTTP/1.1\r\n" 1827 "Connection: close\r\n" 1828 "Transfer-Encoding: chunked\r\n" 1829 "Host: example.com\r\n" 1830 "\r\n") 1831 self.transport.clear() 1832 1833 producer.consumer.write('x' * 3) 1834 producer.consumer.write('y' * 15) 1835 producer.finished.callback(None) 1836 self.assertIdentical(self.transport.producer, None) 1837 self.assertEqual( 1838 self.transport.value(), 1839 "3\r\n" 1840 "xxx\r\n" 1841 "f\r\n" 1842 "yyyyyyyyyyyyyyy\r\n" 1843 "0\r\n" 1844 "\r\n") 1845 1846 1847 def test_sendChunkedRequestBodyWithError(self): 1848 """ 1849 If L{Request} is created with a C{bodyProducer} without a known length 1850 and the L{Deferred} returned from its C{startProducing} method fires 1851 with a L{Failure}, the L{Deferred} returned by L{Request.writeTo} fires 1852 with that L{Failure} and the body producer is unregistered from the 1853 transport. The final zero-length chunk is not written to the 1854 transport. 1855 """ 1856 producer = StringProducer(UNKNOWN_LENGTH) 1857 request = Request('POST', '/bar', _boringHeaders, producer) 1858 writeDeferred = request.writeTo(self.transport) 1859 self.transport.clear() 1860 producer.finished.errback(ArbitraryException()) 1861 def cbFailed(ignored): 1862 self.assertEqual(self.transport.value(), "") 1863 self.assertIdentical(self.transport.producer, None) 1864 d = self.assertFailure(writeDeferred, ArbitraryException) 1865 d.addCallback(cbFailed) 1866 return d 1867 1868 1869 def test_sendRequestBodyWithLength(self): 1870 """ 1871 If L{Request} is created with a C{bodyProducer} with a known length, 1872 that length is sent as the value for the I{Content-Length} header and 1873 chunked encoding is not used. 1874 """ 1875 producer = StringProducer(3) 1876 request = Request('POST', '/bar', _boringHeaders, producer) 1877 request.writeTo(self.transport) 1878 1879 self.assertNotIdentical(producer.consumer, None) 1880 self.assertIdentical(self.transport.producer, producer) 1881 self.assertTrue(self.transport.streaming) 1882 1883 self.assertEqual( 1884 self.transport.value(), 1885 "POST /bar HTTP/1.1\r\n" 1886 "Connection: close\r\n" 1887 "Content-Length: 3\r\n" 1888 "Host: example.com\r\n" 1889 "\r\n") 1890 self.transport.clear() 1891 1892 producer.consumer.write('abc') 1893 producer.finished.callback(None) 1894 self.assertIdentical(self.transport.producer, None) 1895 self.assertEqual(self.transport.value(), "abc") 1896 1897 1898 def test_sendRequestBodyWithTooFewBytes(self): 1899 """ 1900 If L{Request} is created with a C{bodyProducer} with a known length and 1901 the producer does not produce that many bytes, the L{Deferred} returned 1902 by L{Request.writeTo} fires with a L{Failure} wrapping a 1903 L{WrongBodyLength} exception. 1904 """ 1905 producer = StringProducer(3) 1906 request = Request('POST', '/bar', _boringHeaders, producer) 1907 writeDeferred = request.writeTo(self.transport) 1908 producer.consumer.write('ab') 1909 producer.finished.callback(None) 1910 self.assertIdentical(self.transport.producer, None) 1911 return self.assertFailure(writeDeferred, WrongBodyLength) 1912 1913 1914 def _sendRequestBodyWithTooManyBytesTest(self, finisher): 1915 """ 1916 Verify that when too many bytes have been written by a body producer 1917 and then the body producer's C{startProducing} L{Deferred} fires that 1918 the producer is unregistered from the transport and that the 1919 L{Deferred} returned from L{Request.writeTo} is fired with a L{Failure} 1920 wrapping a L{WrongBodyLength}. 1921 1922 @param finisher: A callable which will be invoked with the body 1923 producer after too many bytes have been written to the transport. 1924 It should fire the startProducing Deferred somehow. 1925 """ 1926 producer = StringProducer(3) 1927 request = Request('POST', '/bar', _boringHeaders, producer) 1928 writeDeferred = request.writeTo(self.transport) 1929 1930 producer.consumer.write('ab') 1931 1932 # The producer hasn't misbehaved yet, so it shouldn't have been 1933 # stopped. 1934 self.assertFalse(producer.stopped) 1935 1936 producer.consumer.write('cd') 1937 1938 # Now the producer *has* misbehaved, so we should have tried to 1939 # make it stop. 1940 self.assertTrue(producer.stopped) 1941 1942 # The transport should have had the producer unregistered from it as 1943 # well. 1944 self.assertIdentical(self.transport.producer, None) 1945 1946 def cbFailed(exc): 1947 # The "cd" should not have been written to the transport because 1948 # the request can now locally be recognized to be invalid. If we 1949 # had written the extra bytes, the server could have decided to 1950 # start processing the request, which would be bad since we're 1951 # going to indicate failure locally. 1952 self.assertEqual( 1953 self.transport.value(), 1954 "POST /bar HTTP/1.1\r\n" 1955 "Connection: close\r\n" 1956 "Content-Length: 3\r\n" 1957 "Host: example.com\r\n" 1958 "\r\n" 1959 "ab") 1960 self.transport.clear() 1961 1962 # Subsequent writes should be ignored, as should firing the 1963 # Deferred returned from startProducing. 1964 self.assertRaises(ExcessWrite, producer.consumer.write, 'ef') 1965 1966 # Likewise, if the Deferred returned from startProducing fires, 1967 # this should more or less be ignored (aside from possibly logging 1968 # an error). 1969 finisher(producer) 1970 1971 # There should have been nothing further written to the transport. 1972 self.assertEqual(self.transport.value(), "") 1973 1974 d = self.assertFailure(writeDeferred, WrongBodyLength) 1975 d.addCallback(cbFailed) 1976 return d 1977 1978 1979 def test_sendRequestBodyWithTooManyBytes(self): 1980 """ 1981 If L{Request} is created with a C{bodyProducer} with a known length and 1982 the producer tries to produce more than than many bytes, the 1983 L{Deferred} returned by L{Request.writeTo} fires with a L{Failure} 1984 wrapping a L{WrongBodyLength} exception. 1985 """ 1986 def finisher(producer): 1987 producer.finished.callback(None) 1988 return self._sendRequestBodyWithTooManyBytesTest(finisher) 1989 1990 1991 def test_sendRequestBodyErrorWithTooManyBytes(self): 1992 """ 1993 If L{Request} is created with a C{bodyProducer} with a known length and 1994 the producer tries to produce more than than many bytes, the 1995 L{Deferred} returned by L{Request.writeTo} fires with a L{Failure} 1996 wrapping a L{WrongBodyLength} exception. 1997 """ 1998 def finisher(producer): 1999 producer.finished.errback(ArbitraryException()) 2000 errors = self.flushLoggedErrors(ArbitraryException) 2001 self.assertEqual(len(errors), 1) 2002 return self._sendRequestBodyWithTooManyBytesTest(finisher) 2003 2004 2005 def test_sendRequestBodyErrorWithConsumerError(self): 2006 """ 2007 Though there should be no way for the internal C{finishedConsuming} 2008 L{Deferred} in L{Request._writeToContentLength} to fire a L{Failure} 2009 after the C{finishedProducing} L{Deferred} has fired, in case this does 2010 happen, the error should be logged with a message about how there's 2011 probably a bug in L{Request}. 2012 2013 This is a whitebox test. 2014 """ 2015 producer = StringProducer(3) 2016 request = Request('POST', '/bar', _boringHeaders, producer) 2017 request.writeTo(self.transport) 2018 2019 finishedConsuming = producer.consumer._finished 2020 2021 producer.consumer.write('abc') 2022 producer.finished.callback(None) 2023 2024 finishedConsuming.errback(ArbitraryException()) 2025 self.assertEqual(len(self.flushLoggedErrors(ArbitraryException)), 1) 2026 2027 2028 def _sendRequestBodyFinishedEarlyThenTooManyBytes(self, finisher): 2029 """ 2030 Verify that if the body producer fires its Deferred and then keeps 2031 writing to the consumer that the extra writes are ignored and the 2032 L{Deferred} returned by L{Request.writeTo} fires with a L{Failure} 2033 wrapping the most appropriate exception type. 2034 """ 2035 producer = StringProducer(3) 2036 request = Request('POST', '/bar', _boringHeaders, producer) 2037 writeDeferred = request.writeTo(self.transport) 2038 2039 producer.consumer.write('ab') 2040 finisher(producer) 2041 self.assertIdentical(self.transport.producer, None) 2042 self.transport.clear() 2043 self.assertRaises(ExcessWrite, producer.consumer.write, 'cd') 2044 self.assertEqual(self.transport.value(), "") 2045 return writeDeferred 2046 2047 2048 def test_sendRequestBodyFinishedEarlyThenTooManyBytes(self): 2049 """ 2050 If the request body producer indicates it is done by firing the 2051 L{Deferred} returned from its C{startProducing} method but then goes on 2052 to write too many bytes, the L{Deferred} returned by {Request.writeTo} 2053 fires with a L{Failure} wrapping L{WrongBodyLength}. 2054 """ 2055 def finisher(producer): 2056 producer.finished.callback(None) 2057 return self.assertFailure( 2058 self._sendRequestBodyFinishedEarlyThenTooManyBytes(finisher), 2059 WrongBodyLength) 2060 2061 2062 def test_sendRequestBodyErroredEarlyThenTooManyBytes(self): 2063 """ 2064 If the request body producer indicates an error by firing the 2065 L{Deferred} returned from its C{startProducing} method but then goes on 2066 to write too many bytes, the L{Deferred} returned by {Request.writeTo} 2067 fires with that L{Failure} and L{WrongBodyLength} is logged. 2068 """ 2069 def finisher(producer): 2070 producer.finished.errback(ArbitraryException()) 2071 return self.assertFailure( 2072 self._sendRequestBodyFinishedEarlyThenTooManyBytes(finisher), 2073 ArbitraryException) 2074 2075 2076 def test_sendChunkedRequestBodyFinishedThenWriteMore(self, _with=None): 2077 """ 2078 If the request body producer with an unknown length tries to write 2079 after firing the L{Deferred} returned by its C{startProducing} method, 2080 the C{write} call raises an exception and does not write anything to 2081 the underlying transport. 2082 """ 2083 producer = StringProducer(UNKNOWN_LENGTH) 2084 request = Request('POST', '/bar', _boringHeaders, producer) 2085 writeDeferred = request.writeTo(self.transport) 2086 producer.finished.callback(_with) 2087 self.transport.clear() 2088 2089 self.assertRaises(ExcessWrite, producer.consumer.write, 'foo') 2090 self.assertEqual(self.transport.value(), "") 2091 return writeDeferred 2092 2093 2094 def test_sendChunkedRequestBodyFinishedWithErrorThenWriteMore(self): 2095 """ 2096 If the request body producer with an unknown length tries to write 2097 after firing the L{Deferred} returned by its C{startProducing} method 2098 with a L{Failure}, the C{write} call raises an exception and does not 2099 write anything to the underlying transport. 2100 """ 2101 d = self.test_sendChunkedRequestBodyFinishedThenWriteMore( 2102 Failure(ArbitraryException())) 2103 return self.assertFailure(d, ArbitraryException) 2104 2105 2106 def test_sendRequestBodyWithError(self): 2107 """ 2108 If the L{Deferred} returned from the C{startProducing} method of the 2109 L{IBodyProducer} passed to L{Request} fires with a L{Failure}, the 2110 L{Deferred} returned from L{Request.writeTo} fails with that 2111 L{Failure}. 2112 """ 2113 producer = StringProducer(5) 2114 request = Request('POST', '/bar', _boringHeaders, producer) 2115 writeDeferred = request.writeTo(self.transport) 2116 2117 # Sanity check - the producer should be registered with the underlying 2118 # transport. 2119 self.assertIdentical(self.transport.producer, producer) 2120 self.assertTrue(self.transport.streaming) 2121 2122 producer.consumer.write('ab') 2123 self.assertEqual( 2124 self.transport.value(), 2125 "POST /bar HTTP/1.1\r\n" 2126 "Connection: close\r\n" 2127 "Content-Length: 5\r\n" 2128 "Host: example.com\r\n" 2129 "\r\n" 2130 "ab") 2131 2132 self.assertFalse(self.transport.disconnecting) 2133 producer.finished.errback(Failure(ArbitraryException())) 2134 2135 # Disconnection is handled by a higher level. Request should leave the 2136 # transport alone in this case. 2137 self.assertFalse(self.transport.disconnecting) 2138 2139 # Oh. Except it should unregister the producer that it registered. 2140 self.assertIdentical(self.transport.producer, None) 2141 2142 return self.assertFailure(writeDeferred, ArbitraryException) 2143 2144 2145 def test_hostHeaderRequired(self): 2146 """ 2147 L{Request.writeTo} raises L{BadHeaders} if there is not exactly one 2148 I{Host} header and writes nothing to the given transport. 2149 """ 2150 request = Request('GET', '/', Headers({}), None) 2151 self.assertRaises(BadHeaders, request.writeTo, self.transport) 2152 self.assertEqual(self.transport.value(), '') 2153 2154 request = Request('GET', '/', Headers({'Host': ['example.com', 'example.org']}), None) 2155 self.assertRaises(BadHeaders, request.writeTo, self.transport) 2156 self.assertEqual(self.transport.value(), '') 2157 2158 2159 def test_stopWriting(self): 2160 """ 2161 L{Request.stopWriting} calls its body producer's C{stopProducing} 2162 method. 2163 """ 2164 producer = StringProducer(3) 2165 request = Request('GET', '/', _boringHeaders, producer) 2166 request.writeTo(self.transport) 2167 self.assertFalse(producer.stopped) 2168 request.stopWriting() 2169 self.assertTrue(producer.stopped) 2170 2171 2172 def test_brokenStopProducing(self): 2173 """ 2174 If the body producer's C{stopProducing} method raises an exception, 2175 L{Request.stopWriting} logs it and does not re-raise it. 2176 """ 2177 producer = StringProducer(3) 2178 def brokenStopProducing(): 2179 raise ArbitraryException("stopProducing is busted") 2180 producer.stopProducing = brokenStopProducing 2181 2182 request = Request('GET', '/', _boringHeaders, producer) 2183 request.writeTo(self.transport) 2184 request.stopWriting() 2185 self.assertEqual( 2186 len(self.flushLoggedErrors(ArbitraryException)), 1) 2187 2188 2189 2190class LengthEnforcingConsumerTests(TestCase): 2191 """ 2192 Tests for L{LengthEnforcingConsumer}. 2193 """ 2194 def setUp(self): 2195 self.result = Deferred() 2196 self.producer = StringProducer(10) 2197 self.transport = StringTransport() 2198 self.enforcer = LengthEnforcingConsumer( 2199 self.producer, self.transport, self.result) 2200 2201 2202 def test_write(self): 2203 """ 2204 L{LengthEnforcingConsumer.write} calls the wrapped consumer's C{write} 2205 method with the bytes it is passed as long as there are fewer of them 2206 than the C{length} attribute indicates remain to be received. 2207 """ 2208 self.enforcer.write('abc') 2209 self.assertEqual(self.transport.value(), 'abc') 2210 self.transport.clear() 2211 self.enforcer.write('def') 2212 self.assertEqual(self.transport.value(), 'def') 2213 2214 2215 def test_finishedEarly(self): 2216 """ 2217 L{LengthEnforcingConsumer._noMoreWritesExpected} raises 2218 L{WrongBodyLength} if it is called before the indicated number of bytes 2219 have been written. 2220 """ 2221 self.enforcer.write('x' * 9) 2222 self.assertRaises(WrongBodyLength, self.enforcer._noMoreWritesExpected) 2223 2224 2225 def test_writeTooMany(self, _unregisterAfter=False): 2226 """ 2227 If it is called with a total number of bytes exceeding the indicated 2228 limit passed to L{LengthEnforcingConsumer.__init__}, 2229 L{LengthEnforcingConsumer.write} fires the L{Deferred} with a 2230 L{Failure} wrapping a L{WrongBodyLength} and also calls the 2231 C{stopProducing} method of the producer. 2232 """ 2233 self.enforcer.write('x' * 10) 2234 self.assertFalse(self.producer.stopped) 2235 self.enforcer.write('x') 2236 self.assertTrue(self.producer.stopped) 2237 if _unregisterAfter: 2238 self.enforcer._noMoreWritesExpected() 2239 return self.assertFailure(self.result, WrongBodyLength) 2240 2241 2242 def test_writeAfterNoMoreExpected(self): 2243 """ 2244 If L{LengthEnforcingConsumer.write} is called after 2245 L{LengthEnforcingConsumer._noMoreWritesExpected}, it calls the 2246 producer's C{stopProducing} method and raises L{ExcessWrite}. 2247 """ 2248 self.enforcer.write('x' * 10) 2249 self.enforcer._noMoreWritesExpected() 2250 self.assertFalse(self.producer.stopped) 2251 self.assertRaises(ExcessWrite, self.enforcer.write, 'x') 2252 self.assertTrue(self.producer.stopped) 2253 2254 2255 def test_finishedLate(self): 2256 """ 2257 L{LengthEnforcingConsumer._noMoreWritesExpected} does nothing (in 2258 particular, it does not raise any exception) if called after too many 2259 bytes have been passed to C{write}. 2260 """ 2261 return self.test_writeTooMany(True) 2262 2263 2264 def test_finished(self): 2265 """ 2266 If L{LengthEnforcingConsumer._noMoreWritesExpected} is called after 2267 the correct number of bytes have been written it returns C{None}. 2268 """ 2269 self.enforcer.write('x' * 10) 2270 self.assertIdentical(self.enforcer._noMoreWritesExpected(), None) 2271 2272 2273 def test_stopProducingRaises(self): 2274 """ 2275 If L{LengthEnforcingConsumer.write} calls the producer's 2276 C{stopProducing} because too many bytes were written and the 2277 C{stopProducing} method raises an exception, the exception is logged 2278 and the L{LengthEnforcingConsumer} still errbacks the finished 2279 L{Deferred}. 2280 """ 2281 def brokenStopProducing(): 2282 StringProducer.stopProducing(self.producer) 2283 raise ArbitraryException("stopProducing is busted") 2284 self.producer.stopProducing = brokenStopProducing 2285 2286 def cbFinished(ignored): 2287 self.assertEqual( 2288 len(self.flushLoggedErrors(ArbitraryException)), 1) 2289 d = self.test_writeTooMany() 2290 d.addCallback(cbFinished) 2291 return d 2292 2293 2294 2295class RequestBodyConsumerTests(TestCase): 2296 """ 2297 Tests for L{ChunkedEncoder} which sits between an L{ITransport} and a 2298 request/response body producer and chunked encodes everything written to 2299 it. 2300 """ 2301 def test_interface(self): 2302 """ 2303 L{ChunkedEncoder} instances provide L{IConsumer}. 2304 """ 2305 self.assertTrue( 2306 verifyObject(IConsumer, ChunkedEncoder(StringTransport()))) 2307 2308 2309 def test_write(self): 2310 """ 2311 L{ChunkedEncoder.write} writes to the transport the chunked encoded 2312 form of the bytes passed to it. 2313 """ 2314 transport = StringTransport() 2315 encoder = ChunkedEncoder(transport) 2316 encoder.write('foo') 2317 self.assertEqual(transport.value(), '3\r\nfoo\r\n') 2318 transport.clear() 2319 encoder.write('x' * 16) 2320 self.assertEqual(transport.value(), '10\r\n' + 'x' * 16 + '\r\n') 2321 2322 2323 def test_producerRegistration(self): 2324 """ 2325 L{ChunkedEncoder.registerProducer} registers the given streaming 2326 producer with its transport and L{ChunkedEncoder.unregisterProducer} 2327 writes a zero-length chunk to its transport and unregisters the 2328 transport's producer. 2329 """ 2330 transport = StringTransport() 2331 producer = object() 2332 encoder = ChunkedEncoder(transport) 2333 encoder.registerProducer(producer, True) 2334 self.assertIdentical(transport.producer, producer) 2335 self.assertTrue(transport.streaming) 2336 encoder.unregisterProducer() 2337 self.assertIdentical(transport.producer, None) 2338 self.assertEqual(transport.value(), '0\r\n\r\n') 2339 2340 2341 2342class TransportProxyProducerTests(TestCase): 2343 """ 2344 Tests for L{TransportProxyProducer} which proxies the L{IPushProducer} 2345 interface of a transport. 2346 """ 2347 def test_interface(self): 2348 """ 2349 L{TransportProxyProducer} instances provide L{IPushProducer}. 2350 """ 2351 self.assertTrue( 2352 verifyObject(IPushProducer, TransportProxyProducer(None))) 2353 2354 2355 def test_stopProxyingUnreferencesProducer(self): 2356 """ 2357 L{TransportProxyProducer._stopProxying} drops the reference to the 2358 wrapped L{IPushProducer} provider. 2359 """ 2360 transport = StringTransport() 2361 proxy = TransportProxyProducer(transport) 2362 self.assertIdentical(proxy._producer, transport) 2363 proxy._stopProxying() 2364 self.assertIdentical(proxy._producer, None) 2365 2366 2367 def test_resumeProducing(self): 2368 """ 2369 L{TransportProxyProducer.resumeProducing} calls the wrapped 2370 transport's C{resumeProducing} method unless told to stop proxying. 2371 """ 2372 transport = StringTransport() 2373 transport.pauseProducing() 2374 2375 proxy = TransportProxyProducer(transport) 2376 # The transport should still be paused. 2377 self.assertEqual(transport.producerState, 'paused') 2378 proxy.resumeProducing() 2379 # The transport should now be resumed. 2380 self.assertEqual(transport.producerState, 'producing') 2381 2382 transport.pauseProducing() 2383 proxy._stopProxying() 2384 2385 # The proxy should no longer do anything to the transport. 2386 proxy.resumeProducing() 2387 self.assertEqual(transport.producerState, 'paused') 2388 2389 2390 def test_pauseProducing(self): 2391 """ 2392 L{TransportProxyProducer.pauseProducing} calls the wrapped transport's 2393 C{pauseProducing} method unless told to stop proxying. 2394 """ 2395 transport = StringTransport() 2396 2397 proxy = TransportProxyProducer(transport) 2398 # The transport should still be producing. 2399 self.assertEqual(transport.producerState, 'producing') 2400 proxy.pauseProducing() 2401 # The transport should now be paused. 2402 self.assertEqual(transport.producerState, 'paused') 2403 2404 transport.resumeProducing() 2405 proxy._stopProxying() 2406 2407 # The proxy should no longer do anything to the transport. 2408 proxy.pauseProducing() 2409 self.assertEqual(transport.producerState, 'producing') 2410 2411 2412 def test_stopProducing(self): 2413 """ 2414 L{TransportProxyProducer.stopProducing} calls the wrapped transport's 2415 C{stopProducing} method unless told to stop proxying. 2416 """ 2417 transport = StringTransport() 2418 proxy = TransportProxyProducer(transport) 2419 # The transport should still be producing. 2420 self.assertEqual(transport.producerState, 'producing') 2421 proxy.stopProducing() 2422 # The transport should now be stopped. 2423 self.assertEqual(transport.producerState, 'stopped') 2424 2425 transport = StringTransport() 2426 proxy = TransportProxyProducer(transport) 2427 proxy._stopProxying() 2428 proxy.stopProducing() 2429 # The transport should not have been stopped. 2430 self.assertEqual(transport.producerState, 'producing') 2431 2432 2433 2434class ResponseTests(TestCase): 2435 """ 2436 Tests for L{Response}. 2437 """ 2438 2439 def test_verifyInterface(self): 2440 """ 2441 L{Response} instances provide L{IResponse}. 2442 """ 2443 response = justTransportResponse(StringTransport()) 2444 self.assertTrue(verifyObject(IResponse, response)) 2445 2446 2447 def test_makeConnection(self): 2448 """ 2449 The L{IProtocol} provider passed to L{Response.deliverBody} has its 2450 C{makeConnection} method called with an L{IPushProducer} provider 2451 hooked up to the response as an argument. 2452 """ 2453 producers = [] 2454 transport = StringTransport() 2455 class SomeProtocol(Protocol): 2456 def makeConnection(self, producer): 2457 producers.append(producer) 2458 2459 consumer = SomeProtocol() 2460 response = justTransportResponse(transport) 2461 response.deliverBody(consumer) 2462 [theProducer] = producers 2463 theProducer.pauseProducing() 2464 self.assertEqual(transport.producerState, 'paused') 2465 theProducer.resumeProducing() 2466 self.assertEqual(transport.producerState, 'producing') 2467 2468 2469 def test_dataReceived(self): 2470 """ 2471 The L{IProtocol} provider passed to L{Response.deliverBody} has its 2472 C{dataReceived} method called with bytes received as part of the 2473 response body. 2474 """ 2475 bytes = [] 2476 class ListConsumer(Protocol): 2477 def dataReceived(self, data): 2478 bytes.append(data) 2479 2480 2481 consumer = ListConsumer() 2482 response = justTransportResponse(StringTransport()) 2483 response.deliverBody(consumer) 2484 2485 response._bodyDataReceived('foo') 2486 self.assertEqual(bytes, ['foo']) 2487 2488 2489 def test_connectionLost(self): 2490 """ 2491 The L{IProtocol} provider passed to L{Response.deliverBody} has its 2492 C{connectionLost} method called with a L{Failure} wrapping 2493 L{ResponseDone} when the response's C{_bodyDataFinished} method is 2494 called. 2495 """ 2496 lost = [] 2497 class ListConsumer(Protocol): 2498 def connectionLost(self, reason): 2499 lost.append(reason) 2500 2501 consumer = ListConsumer() 2502 response = justTransportResponse(StringTransport()) 2503 response.deliverBody(consumer) 2504 2505 response._bodyDataFinished() 2506 lost[0].trap(ResponseDone) 2507 self.assertEqual(len(lost), 1) 2508 2509 # The protocol reference should be dropped, too, to facilitate GC or 2510 # whatever. 2511 self.assertIdentical(response._bodyProtocol, None) 2512 2513 2514 def test_bufferEarlyData(self): 2515 """ 2516 If data is delivered to the L{Response} before a protocol is registered 2517 with C{deliverBody}, that data is buffered until the protocol is 2518 registered and then is delivered. 2519 """ 2520 bytes = [] 2521 class ListConsumer(Protocol): 2522 def dataReceived(self, data): 2523 bytes.append(data) 2524 2525 protocol = ListConsumer() 2526 response = justTransportResponse(StringTransport()) 2527 response._bodyDataReceived('foo') 2528 response._bodyDataReceived('bar') 2529 response.deliverBody(protocol) 2530 response._bodyDataReceived('baz') 2531 self.assertEqual(bytes, ['foo', 'bar', 'baz']) 2532 # Make sure the implementation-detail-byte-buffer is cleared because 2533 # not clearing it wastes memory. 2534 self.assertIdentical(response._bodyBuffer, None) 2535 2536 2537 def test_multipleStartProducingFails(self): 2538 """ 2539 L{Response.deliverBody} raises L{RuntimeError} if called more than 2540 once. 2541 """ 2542 response = justTransportResponse(StringTransport()) 2543 response.deliverBody(Protocol()) 2544 self.assertRaises(RuntimeError, response.deliverBody, Protocol()) 2545 2546 2547 def test_startProducingAfterFinishedFails(self): 2548 """ 2549 L{Response.deliverBody} raises L{RuntimeError} if called after 2550 L{Response._bodyDataFinished}. 2551 """ 2552 response = justTransportResponse(StringTransport()) 2553 response.deliverBody(Protocol()) 2554 response._bodyDataFinished() 2555 self.assertRaises(RuntimeError, response.deliverBody, Protocol()) 2556 2557 2558 def test_bodyDataReceivedAfterFinishedFails(self): 2559 """ 2560 L{Response._bodyDataReceived} raises L{RuntimeError} if called after 2561 L{Response._bodyDataFinished} but before L{Response.deliverBody}. 2562 """ 2563 response = justTransportResponse(StringTransport()) 2564 response._bodyDataFinished() 2565 self.assertRaises(RuntimeError, response._bodyDataReceived, 'foo') 2566 2567 2568 def test_bodyDataReceivedAfterDeliveryFails(self): 2569 """ 2570 L{Response._bodyDataReceived} raises L{RuntimeError} if called after 2571 L{Response._bodyDataFinished} and after L{Response.deliverBody}. 2572 """ 2573 response = justTransportResponse(StringTransport()) 2574 response._bodyDataFinished() 2575 response.deliverBody(Protocol()) 2576 self.assertRaises(RuntimeError, response._bodyDataReceived, 'foo') 2577 2578 2579 def test_bodyDataFinishedAfterFinishedFails(self): 2580 """ 2581 L{Response._bodyDataFinished} raises L{RuntimeError} if called more 2582 than once. 2583 """ 2584 response = justTransportResponse(StringTransport()) 2585 response._bodyDataFinished() 2586 self.assertRaises(RuntimeError, response._bodyDataFinished) 2587 2588 2589 def test_bodyDataFinishedAfterDeliveryFails(self): 2590 """ 2591 L{Response._bodyDataFinished} raises L{RuntimeError} if called after 2592 the body has been delivered. 2593 """ 2594 response = justTransportResponse(StringTransport()) 2595 response._bodyDataFinished() 2596 response.deliverBody(Protocol()) 2597 self.assertRaises(RuntimeError, response._bodyDataFinished) 2598 2599 2600 def test_transportResumed(self): 2601 """ 2602 L{Response.deliverBody} resumes the HTTP connection's transport 2603 before passing it to the transport's C{makeConnection} method. 2604 """ 2605 transportState = [] 2606 class ListConsumer(Protocol): 2607 def makeConnection(self, transport): 2608 transportState.append(transport.producerState) 2609 2610 transport = StringTransport() 2611 transport.pauseProducing() 2612 protocol = ListConsumer() 2613 response = justTransportResponse(transport) 2614 self.assertEqual(transport.producerState, 'paused') 2615 response.deliverBody(protocol) 2616 self.assertEqual(transportState, ['producing']) 2617 2618 2619 def test_bodyDataFinishedBeforeStartProducing(self): 2620 """ 2621 If the entire body is delivered to the L{Response} before the 2622 response's C{deliverBody} method is called, the protocol passed to 2623 C{deliverBody} is immediately given the body data and then 2624 disconnected. 2625 """ 2626 transport = StringTransport() 2627 response = justTransportResponse(transport) 2628 response._bodyDataReceived('foo') 2629 response._bodyDataReceived('bar') 2630 response._bodyDataFinished() 2631 2632 protocol = AccumulatingProtocol() 2633 response.deliverBody(protocol) 2634 self.assertEqual(protocol.data, 'foobar') 2635 protocol.closedReason.trap(ResponseDone) 2636 2637 2638 def test_finishedWithErrorWhenConnected(self): 2639 """ 2640 The L{Failure} passed to L{Response._bodyDataFinished} when the response 2641 is in the I{connected} state is passed to the C{connectionLost} method 2642 of the L{IProtocol} provider passed to the L{Response}'s 2643 C{deliverBody} method. 2644 """ 2645 transport = StringTransport() 2646 response = justTransportResponse(transport) 2647 2648 protocol = AccumulatingProtocol() 2649 response.deliverBody(protocol) 2650 2651 # Sanity check - this test is for the connected state 2652 self.assertEqual(response._state, 'CONNECTED') 2653 response._bodyDataFinished(Failure(ArbitraryException())) 2654 2655 protocol.closedReason.trap(ArbitraryException) 2656 2657 2658 def test_finishedWithErrorWhenInitial(self): 2659 """ 2660 The L{Failure} passed to L{Response._bodyDataFinished} when the response 2661 is in the I{initial} state is passed to the C{connectionLost} method of 2662 the L{IProtocol} provider passed to the L{Response}'s C{deliverBody} 2663 method. 2664 """ 2665 transport = StringTransport() 2666 response = justTransportResponse(transport) 2667 2668 # Sanity check - this test is for the initial state 2669 self.assertEqual(response._state, 'INITIAL') 2670 response._bodyDataFinished(Failure(ArbitraryException())) 2671 2672 protocol = AccumulatingProtocol() 2673 response.deliverBody(protocol) 2674 2675 protocol.closedReason.trap(ArbitraryException) 2676