1""" 2A high-speed, production ready, thread pooled, generic HTTP server. 3 4For those of you wanting to understand internals of this module, here's the 5basic call flow. The server's listening thread runs a very tight loop, 6sticking incoming connections onto a Queue:: 7 8 server = HTTPServer(...) 9 server.start() 10 -> serve() 11 while ready: 12 _connections.run() 13 while not stop_requested: 14 child = socket.accept() # blocks until a request comes in 15 conn = HTTPConnection(child, ...) 16 server.process_conn(conn) # adds conn to threadpool 17 18Worker threads are kept in a pool and poll the Queue, popping off and then 19handling each connection in turn. Each connection can consist of an arbitrary 20number of requests and their responses, so we run a nested loop:: 21 22 while True: 23 conn = server.requests.get() 24 conn.communicate() 25 -> while True: 26 req = HTTPRequest(...) 27 req.parse_request() 28 -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" 29 req.rfile.readline() 30 read_headers(req.rfile, req.inheaders) 31 req.respond() 32 -> response = app(...) 33 try: 34 for chunk in response: 35 if chunk: 36 req.write(chunk) 37 finally: 38 if hasattr(response, "close"): 39 response.close() 40 if req.close_connection: 41 return 42 43For running a server you can invoke :func:`start() <HTTPServer.start()>` (it 44will run the server forever) or use invoking :func:`prepare() 45<HTTPServer.prepare()>` and :func:`serve() <HTTPServer.serve()>` like this:: 46 47 server = HTTPServer(...) 48 server.prepare() 49 try: 50 threading.Thread(target=server.serve).start() 51 52 # waiting/detecting some appropriate stop condition here 53 ... 54 55 finally: 56 server.stop() 57 58And now for a trivial doctest to exercise the test suite 59 60>>> 'HTTPServer' in globals() 61True 62""" 63 64from __future__ import absolute_import, division, print_function 65__metaclass__ = type 66 67import os 68import io 69import re 70import email.utils 71import socket 72import sys 73import time 74import traceback as traceback_ 75import logging 76import platform 77import contextlib 78import threading 79 80try: 81 from functools import lru_cache 82except ImportError: 83 from backports.functools_lru_cache import lru_cache 84 85import six 86from six.moves import queue 87from six.moves import urllib 88 89from . import connections, errors, __version__ 90from ._compat import bton, ntou 91from ._compat import IS_PPC 92from .workers import threadpool 93from .makefile import MakeFile, StreamWriter 94 95 96__all__ = ( 97 'HTTPRequest', 'HTTPConnection', 'HTTPServer', 98 'HeaderReader', 'DropUnderscoreHeaderReader', 99 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', 100 'Gateway', 'get_ssl_adapter_class', 101) 102 103 104IS_WINDOWS = platform.system() == 'Windows' 105"""Flag indicating whether the app is running under Windows.""" 106 107 108IS_GAE = os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/') 109"""Flag indicating whether the app is running in GAE env. 110 111Ref: 112https://cloud.google.com/appengine/docs/standard/python/tools 113/using-local-server#detecting_application_runtime_environment 114""" 115 116 117IS_UID_GID_RESOLVABLE = not IS_WINDOWS and not IS_GAE 118"""Indicates whether UID/GID resolution's available under current platform.""" 119 120 121if IS_UID_GID_RESOLVABLE: 122 try: 123 import grp 124 import pwd 125 except ImportError: 126 """Unavailable in the current env. 127 128 This shouldn't be happening normally. 129 All of the known cases are excluded via the if clause. 130 """ 131 IS_UID_GID_RESOLVABLE = False 132 grp, pwd = None, None 133 import struct 134 135 136if IS_WINDOWS and hasattr(socket, 'AF_INET6'): 137 if not hasattr(socket, 'IPPROTO_IPV6'): 138 socket.IPPROTO_IPV6 = 41 139 if not hasattr(socket, 'IPV6_V6ONLY'): 140 socket.IPV6_V6ONLY = 27 141 142 143if not hasattr(socket, 'SO_PEERCRED'): 144 """ 145 NOTE: the value for SO_PEERCRED can be architecture specific, in 146 which case the getsockopt() will hopefully fail. The arch 147 specific value could be derived from platform.processor() 148 """ 149 socket.SO_PEERCRED = 21 if IS_PPC else 17 150 151 152LF = b'\n' 153CRLF = b'\r\n' 154TAB = b'\t' 155SPACE = b' ' 156COLON = b':' 157SEMICOLON = b';' 158EMPTY = b'' 159ASTERISK = b'*' 160FORWARD_SLASH = b'/' 161QUOTED_SLASH = b'%2F' 162QUOTED_SLASH_REGEX = re.compile(b''.join((b'(?i)', QUOTED_SLASH))) 163 164 165_STOPPING_FOR_INTERRUPT = object() # sentinel used during shutdown 166 167 168comma_separated_headers = [ 169 b'Accept', b'Accept-Charset', b'Accept-Encoding', 170 b'Accept-Language', b'Accept-Ranges', b'Allow', b'Cache-Control', 171 b'Connection', b'Content-Encoding', b'Content-Language', b'Expect', 172 b'If-Match', b'If-None-Match', b'Pragma', b'Proxy-Authenticate', b'TE', 173 b'Trailer', b'Transfer-Encoding', b'Upgrade', b'Vary', b'Via', b'Warning', 174 b'WWW-Authenticate', 175] 176 177 178if not hasattr(logging, 'statistics'): 179 logging.statistics = {} 180 181 182class HeaderReader: 183 """Object for reading headers from an HTTP request. 184 185 Interface and default implementation. 186 """ 187 188 def __call__(self, rfile, hdict=None): # noqa: C901 # FIXME 189 """ 190 Read headers from the given stream into the given header dict. 191 192 If hdict is None, a new header dict is created. Returns the populated 193 header dict. 194 195 Headers which are repeated are folded together using a comma if their 196 specification so dictates. 197 198 This function raises ValueError when the read bytes violate the HTTP 199 spec. 200 You should probably return "400 Bad Request" if this happens. 201 """ 202 if hdict is None: 203 hdict = {} 204 205 while True: 206 line = rfile.readline() 207 if not line: 208 # No more data--illegal end of headers 209 raise ValueError('Illegal end of headers.') 210 211 if line == CRLF: 212 # Normal end of headers 213 break 214 if not line.endswith(CRLF): 215 raise ValueError('HTTP requires CRLF terminators') 216 217 if line[0] in (SPACE, TAB): 218 # It's a continuation line. 219 v = line.strip() 220 else: 221 try: 222 k, v = line.split(COLON, 1) 223 except ValueError: 224 raise ValueError('Illegal header line.') 225 v = v.strip() 226 k = self._transform_key(k) 227 hname = k 228 229 if not self._allow_header(k): 230 continue 231 232 if k in comma_separated_headers: 233 existing = hdict.get(hname) 234 if existing: 235 v = b', '.join((existing, v)) 236 hdict[hname] = v 237 238 return hdict 239 240 def _allow_header(self, key_name): 241 return True 242 243 def _transform_key(self, key_name): 244 # TODO: what about TE and WWW-Authenticate? 245 return key_name.strip().title() 246 247 248class DropUnderscoreHeaderReader(HeaderReader): 249 """Custom HeaderReader to exclude any headers with underscores in them.""" 250 251 def _allow_header(self, key_name): 252 orig = super(DropUnderscoreHeaderReader, self)._allow_header(key_name) 253 return orig and '_' not in key_name 254 255 256class SizeCheckWrapper: 257 """Wraps a file-like object, raising MaxSizeExceeded if too large. 258 259 :param rfile: ``file`` of a limited size 260 :param int maxlen: maximum length of the file being read 261 """ 262 263 def __init__(self, rfile, maxlen): 264 """Initialize SizeCheckWrapper instance.""" 265 self.rfile = rfile 266 self.maxlen = maxlen 267 self.bytes_read = 0 268 269 def _check_length(self): 270 if self.maxlen and self.bytes_read > self.maxlen: 271 raise errors.MaxSizeExceeded() 272 273 def read(self, size=None): 274 """Read a chunk from ``rfile`` buffer and return it. 275 276 :param int size: amount of data to read 277 278 :returns: chunk from ``rfile``, limited by size if specified 279 :rtype: bytes 280 """ 281 data = self.rfile.read(size) 282 self.bytes_read += len(data) 283 self._check_length() 284 return data 285 286 def readline(self, size=None): 287 """Read a single line from ``rfile`` buffer and return it. 288 289 :param int size: minimum amount of data to read 290 291 :returns: one line from ``rfile`` 292 :rtype: bytes 293 """ 294 if size is not None: 295 data = self.rfile.readline(size) 296 self.bytes_read += len(data) 297 self._check_length() 298 return data 299 300 # User didn't specify a size ... 301 # We read the line in chunks to make sure it's not a 100MB line ! 302 res = [] 303 while True: 304 data = self.rfile.readline(256) 305 self.bytes_read += len(data) 306 self._check_length() 307 res.append(data) 308 # See https://github.com/cherrypy/cherrypy/issues/421 309 if len(data) < 256 or data[-1:] == LF: 310 return EMPTY.join(res) 311 312 def readlines(self, sizehint=0): 313 """Read all lines from ``rfile`` buffer and return them. 314 315 :param int sizehint: hint of minimum amount of data to read 316 317 :returns: lines of bytes read from ``rfile`` 318 :rtype: list[bytes] 319 """ 320 # Shamelessly stolen from StringIO 321 total = 0 322 lines = [] 323 line = self.readline(sizehint) 324 while line: 325 lines.append(line) 326 total += len(line) 327 if 0 < sizehint <= total: 328 break 329 line = self.readline(sizehint) 330 return lines 331 332 def close(self): 333 """Release resources allocated for ``rfile``.""" 334 self.rfile.close() 335 336 def __iter__(self): 337 """Return file iterator.""" 338 return self 339 340 def __next__(self): 341 """Generate next file chunk.""" 342 data = next(self.rfile) 343 self.bytes_read += len(data) 344 self._check_length() 345 return data 346 347 next = __next__ 348 349 350class KnownLengthRFile: 351 """Wraps a file-like object, returning an empty string when exhausted. 352 353 :param rfile: ``file`` of a known size 354 :param int content_length: length of the file being read 355 """ 356 357 def __init__(self, rfile, content_length): 358 """Initialize KnownLengthRFile instance.""" 359 self.rfile = rfile 360 self.remaining = content_length 361 362 def read(self, size=None): 363 """Read a chunk from ``rfile`` buffer and return it. 364 365 :param int size: amount of data to read 366 367 :rtype: bytes 368 :returns: chunk from ``rfile``, limited by size if specified 369 """ 370 if self.remaining == 0: 371 return b'' 372 if size is None: 373 size = self.remaining 374 else: 375 size = min(size, self.remaining) 376 377 data = self.rfile.read(size) 378 self.remaining -= len(data) 379 return data 380 381 def readline(self, size=None): 382 """Read a single line from ``rfile`` buffer and return it. 383 384 :param int size: minimum amount of data to read 385 386 :returns: one line from ``rfile`` 387 :rtype: bytes 388 """ 389 if self.remaining == 0: 390 return b'' 391 if size is None: 392 size = self.remaining 393 else: 394 size = min(size, self.remaining) 395 396 data = self.rfile.readline(size) 397 self.remaining -= len(data) 398 return data 399 400 def readlines(self, sizehint=0): 401 """Read all lines from ``rfile`` buffer and return them. 402 403 :param int sizehint: hint of minimum amount of data to read 404 405 :returns: lines of bytes read from ``rfile`` 406 :rtype: list[bytes] 407 """ 408 # Shamelessly stolen from StringIO 409 total = 0 410 lines = [] 411 line = self.readline(sizehint) 412 while line: 413 lines.append(line) 414 total += len(line) 415 if 0 < sizehint <= total: 416 break 417 line = self.readline(sizehint) 418 return lines 419 420 def close(self): 421 """Release resources allocated for ``rfile``.""" 422 self.rfile.close() 423 424 def __iter__(self): 425 """Return file iterator.""" 426 return self 427 428 def __next__(self): 429 """Generate next file chunk.""" 430 data = next(self.rfile) 431 self.remaining -= len(data) 432 return data 433 434 next = __next__ 435 436 437class ChunkedRFile: 438 """Wraps a file-like object, returning an empty string when exhausted. 439 440 This class is intended to provide a conforming wsgi.input value for 441 request entities that have been encoded with the 'chunked' transfer 442 encoding. 443 444 :param rfile: file encoded with the 'chunked' transfer encoding 445 :param int maxlen: maximum length of the file being read 446 :param int bufsize: size of the buffer used to read the file 447 """ 448 449 def __init__(self, rfile, maxlen, bufsize=8192): 450 """Initialize ChunkedRFile instance.""" 451 self.rfile = rfile 452 self.maxlen = maxlen 453 self.bytes_read = 0 454 self.buffer = EMPTY 455 self.bufsize = bufsize 456 self.closed = False 457 458 def _fetch(self): 459 if self.closed: 460 return 461 462 line = self.rfile.readline() 463 self.bytes_read += len(line) 464 465 if self.maxlen and self.bytes_read > self.maxlen: 466 raise errors.MaxSizeExceeded( 467 'Request Entity Too Large', self.maxlen, 468 ) 469 470 line = line.strip().split(SEMICOLON, 1) 471 472 try: 473 chunk_size = line.pop(0) 474 chunk_size = int(chunk_size, 16) 475 except ValueError: 476 raise ValueError( 477 'Bad chunked transfer size: {chunk_size!r}'. 478 format(chunk_size=chunk_size), 479 ) 480 481 if chunk_size <= 0: 482 self.closed = True 483 return 484 485# if line: chunk_extension = line[0] 486 487 if self.maxlen and self.bytes_read + chunk_size > self.maxlen: 488 raise IOError('Request Entity Too Large') 489 490 chunk = self.rfile.read(chunk_size) 491 self.bytes_read += len(chunk) 492 self.buffer += chunk 493 494 crlf = self.rfile.read(2) 495 if crlf != CRLF: 496 raise ValueError( 497 "Bad chunked transfer coding (expected '\\r\\n', " 498 'got ' + repr(crlf) + ')', 499 ) 500 501 def read(self, size=None): 502 """Read a chunk from ``rfile`` buffer and return it. 503 504 :param int size: amount of data to read 505 506 :returns: chunk from ``rfile``, limited by size if specified 507 :rtype: bytes 508 """ 509 data = EMPTY 510 511 if size == 0: 512 return data 513 514 while True: 515 if size and len(data) >= size: 516 return data 517 518 if not self.buffer: 519 self._fetch() 520 if not self.buffer: 521 # EOF 522 return data 523 524 if size: 525 remaining = size - len(data) 526 data += self.buffer[:remaining] 527 self.buffer = self.buffer[remaining:] 528 else: 529 data += self.buffer 530 self.buffer = EMPTY 531 532 def readline(self, size=None): 533 """Read a single line from ``rfile`` buffer and return it. 534 535 :param int size: minimum amount of data to read 536 537 :returns: one line from ``rfile`` 538 :rtype: bytes 539 """ 540 data = EMPTY 541 542 if size == 0: 543 return data 544 545 while True: 546 if size and len(data) >= size: 547 return data 548 549 if not self.buffer: 550 self._fetch() 551 if not self.buffer: 552 # EOF 553 return data 554 555 newline_pos = self.buffer.find(LF) 556 if size: 557 if newline_pos == -1: 558 remaining = size - len(data) 559 data += self.buffer[:remaining] 560 self.buffer = self.buffer[remaining:] 561 else: 562 remaining = min(size - len(data), newline_pos) 563 data += self.buffer[:remaining] 564 self.buffer = self.buffer[remaining:] 565 else: 566 if newline_pos == -1: 567 data += self.buffer 568 self.buffer = EMPTY 569 else: 570 data += self.buffer[:newline_pos] 571 self.buffer = self.buffer[newline_pos:] 572 573 def readlines(self, sizehint=0): 574 """Read all lines from ``rfile`` buffer and return them. 575 576 :param int sizehint: hint of minimum amount of data to read 577 578 :returns: lines of bytes read from ``rfile`` 579 :rtype: list[bytes] 580 """ 581 # Shamelessly stolen from StringIO 582 total = 0 583 lines = [] 584 line = self.readline(sizehint) 585 while line: 586 lines.append(line) 587 total += len(line) 588 if 0 < sizehint <= total: 589 break 590 line = self.readline(sizehint) 591 return lines 592 593 def read_trailer_lines(self): 594 """Read HTTP headers and yield them. 595 596 Returns: 597 Generator: yields CRLF separated lines. 598 599 """ 600 if not self.closed: 601 raise ValueError( 602 'Cannot read trailers until the request body has been read.', 603 ) 604 605 while True: 606 line = self.rfile.readline() 607 if not line: 608 # No more data--illegal end of headers 609 raise ValueError('Illegal end of headers.') 610 611 self.bytes_read += len(line) 612 if self.maxlen and self.bytes_read > self.maxlen: 613 raise IOError('Request Entity Too Large') 614 615 if line == CRLF: 616 # Normal end of headers 617 break 618 if not line.endswith(CRLF): 619 raise ValueError('HTTP requires CRLF terminators') 620 621 yield line 622 623 def close(self): 624 """Release resources allocated for ``rfile``.""" 625 self.rfile.close() 626 627 628class HTTPRequest: 629 """An HTTP Request (and response). 630 631 A single HTTP connection may consist of multiple request/response pairs. 632 """ 633 634 server = None 635 """The HTTPServer object which is receiving this request.""" 636 637 conn = None 638 """The HTTPConnection object on which this request connected.""" 639 640 inheaders = {} 641 """A dict of request headers.""" 642 643 outheaders = [] 644 """A list of header tuples to write in the response.""" 645 646 ready = False 647 """When True, the request has been parsed and is ready to begin generating 648 the response. When False, signals the calling Connection that the response 649 should not be generated and the connection should close.""" 650 651 close_connection = False 652 """Signals the calling Connection that the request should close. This does 653 not imply an error! The client and/or server may each request that the 654 connection be closed.""" 655 656 chunked_write = False 657 """If True, output will be encoded with the "chunked" transfer-coding. 658 659 This value is set automatically inside send_headers.""" 660 661 header_reader = HeaderReader() 662 """ 663 A HeaderReader instance or compatible reader. 664 """ 665 666 def __init__(self, server, conn, proxy_mode=False, strict_mode=True): 667 """Initialize HTTP request container instance. 668 669 Args: 670 server (HTTPServer): web server object receiving this request 671 conn (HTTPConnection): HTTP connection object for this request 672 proxy_mode (bool): whether this HTTPServer should behave as a PROXY 673 server for certain requests 674 strict_mode (bool): whether we should return a 400 Bad Request when 675 we encounter a request that a HTTP compliant client should not be 676 making 677 """ 678 self.server = server 679 self.conn = conn 680 681 self.ready = False 682 self.started_request = False 683 self.scheme = b'http' 684 if self.server.ssl_adapter is not None: 685 self.scheme = b'https' 686 # Use the lowest-common protocol in case read_request_line errors. 687 self.response_protocol = 'HTTP/1.0' 688 self.inheaders = {} 689 690 self.status = '' 691 self.outheaders = [] 692 self.sent_headers = False 693 self.close_connection = self.__class__.close_connection 694 self.chunked_read = False 695 self.chunked_write = self.__class__.chunked_write 696 self.proxy_mode = proxy_mode 697 self.strict_mode = strict_mode 698 699 def parse_request(self): 700 """Parse the next HTTP request start-line and message-headers.""" 701 self.rfile = SizeCheckWrapper( 702 self.conn.rfile, 703 self.server.max_request_header_size, 704 ) 705 try: 706 success = self.read_request_line() 707 except errors.MaxSizeExceeded: 708 self.simple_response( 709 '414 Request-URI Too Long', 710 'The Request-URI sent with the request exceeds the maximum ' 711 'allowed bytes.', 712 ) 713 return 714 else: 715 if not success: 716 return 717 718 try: 719 success = self.read_request_headers() 720 except errors.MaxSizeExceeded: 721 self.simple_response( 722 '413 Request Entity Too Large', 723 'The headers sent with the request exceed the maximum ' 724 'allowed bytes.', 725 ) 726 return 727 else: 728 if not success: 729 return 730 731 self.ready = True 732 733 def read_request_line(self): # noqa: C901 # FIXME 734 """Read and parse first line of the HTTP request. 735 736 Returns: 737 bool: True if the request line is valid or False if it's malformed. 738 739 """ 740 # HTTP/1.1 connections are persistent by default. If a client 741 # requests a page, then idles (leaves the connection open), 742 # then rfile.readline() will raise socket.error("timed out"). 743 # Note that it does this based on the value given to settimeout(), 744 # and doesn't need the client to request or acknowledge the close 745 # (although your TCP stack might suffer for it: cf Apache's history 746 # with FIN_WAIT_2). 747 request_line = self.rfile.readline() 748 749 # Set started_request to True so communicate() knows to send 408 750 # from here on out. 751 self.started_request = True 752 if not request_line: 753 return False 754 755 if request_line == CRLF: 756 # RFC 2616 sec 4.1: "...if the server is reading the protocol 757 # stream at the beginning of a message and receives a CRLF 758 # first, it should ignore the CRLF." 759 # But only ignore one leading line! else we enable a DoS. 760 request_line = self.rfile.readline() 761 if not request_line: 762 return False 763 764 if not request_line.endswith(CRLF): 765 self.simple_response( 766 '400 Bad Request', 'HTTP requires CRLF terminators', 767 ) 768 return False 769 770 try: 771 method, uri, req_protocol = request_line.strip().split(SPACE, 2) 772 if not req_protocol.startswith(b'HTTP/'): 773 self.simple_response( 774 '400 Bad Request', 'Malformed Request-Line: bad protocol', 775 ) 776 return False 777 rp = req_protocol[5:].split(b'.', 1) 778 if len(rp) != 2: 779 self.simple_response( 780 '400 Bad Request', 'Malformed Request-Line: bad version', 781 ) 782 return False 783 rp = tuple(map(int, rp)) # Minor.Major must be threat as integers 784 if rp > (1, 1): 785 self.simple_response( 786 '505 HTTP Version Not Supported', 'Cannot fulfill request', 787 ) 788 return False 789 except (ValueError, IndexError): 790 self.simple_response('400 Bad Request', 'Malformed Request-Line') 791 return False 792 793 self.uri = uri 794 self.method = method.upper() 795 796 if self.strict_mode and method != self.method: 797 resp = ( 798 'Malformed method name: According to RFC 2616 ' 799 '(section 5.1.1) and its successors ' 800 'RFC 7230 (section 3.1.1) and RFC 7231 (section 4.1) ' 801 'method names are case-sensitive and uppercase.' 802 ) 803 self.simple_response('400 Bad Request', resp) 804 return False 805 806 try: 807 if six.PY2: # FIXME: Figure out better way to do this 808 # Ref: https://stackoverflow.com/a/196392/595220 (like this?) 809 """This is a dummy check for unicode in URI.""" 810 ntou(bton(uri, 'ascii'), 'ascii') 811 scheme, authority, path, qs, fragment = urllib.parse.urlsplit(uri) 812 except UnicodeError: 813 self.simple_response('400 Bad Request', 'Malformed Request-URI') 814 return False 815 816 uri_is_absolute_form = (scheme or authority) 817 818 if self.method == b'OPTIONS': 819 # TODO: cover this branch with tests 820 path = ( 821 uri 822 # https://tools.ietf.org/html/rfc7230#section-5.3.4 823 if (self.proxy_mode and uri_is_absolute_form) 824 else path 825 ) 826 elif self.method == b'CONNECT': 827 # TODO: cover this branch with tests 828 if not self.proxy_mode: 829 self.simple_response('405 Method Not Allowed') 830 return False 831 832 # `urlsplit()` above parses "example.com:3128" as path part of URI. 833 # this is a workaround, which makes it detect netloc correctly 834 uri_split = urllib.parse.urlsplit(b''.join((b'//', uri))) 835 _scheme, _authority, _path, _qs, _fragment = uri_split 836 _port = EMPTY 837 try: 838 _port = uri_split.port 839 except ValueError: 840 pass 841 842 # FIXME: use third-party validation to make checks against RFC 843 # the validation doesn't take into account, that urllib parses 844 # invalid URIs without raising errors 845 # https://tools.ietf.org/html/rfc7230#section-5.3.3 846 invalid_path = ( 847 _authority != uri 848 or not _port 849 or any((_scheme, _path, _qs, _fragment)) 850 ) 851 if invalid_path: 852 self.simple_response( 853 '400 Bad Request', 854 'Invalid path in Request-URI: request-' 855 'target must match authority-form.', 856 ) 857 return False 858 859 authority = path = _authority 860 scheme = qs = fragment = EMPTY 861 else: 862 disallowed_absolute = ( 863 self.strict_mode 864 and not self.proxy_mode 865 and uri_is_absolute_form 866 ) 867 if disallowed_absolute: 868 # https://tools.ietf.org/html/rfc7230#section-5.3.2 869 # (absolute form) 870 """Absolute URI is only allowed within proxies.""" 871 self.simple_response( 872 '400 Bad Request', 873 'Absolute URI not allowed if server is not a proxy.', 874 ) 875 return False 876 877 invalid_path = ( 878 self.strict_mode 879 and not uri.startswith(FORWARD_SLASH) 880 and not uri_is_absolute_form 881 ) 882 if invalid_path: 883 # https://tools.ietf.org/html/rfc7230#section-5.3.1 884 # (origin_form) and 885 """Path should start with a forward slash.""" 886 resp = ( 887 'Invalid path in Request-URI: request-target must contain ' 888 'origin-form which starts with absolute-path (URI ' 889 'starting with a slash "/").' 890 ) 891 self.simple_response('400 Bad Request', resp) 892 return False 893 894 if fragment: 895 self.simple_response( 896 '400 Bad Request', 897 'Illegal #fragment in Request-URI.', 898 ) 899 return False 900 901 if path is None: 902 # FIXME: It looks like this case cannot happen 903 self.simple_response( 904 '400 Bad Request', 905 'Invalid path in Request-URI.', 906 ) 907 return False 908 909 # Unquote the path+params (e.g. "/this%20path" -> "/this path"). 910 # https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 911 # 912 # But note that "...a URI must be separated into its components 913 # before the escaped characters within those components can be 914 # safely decoded." https://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 915 # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not 916 # "/this/path". 917 try: 918 # TODO: Figure out whether exception can really happen here. 919 # It looks like it's caught on urlsplit() call above. 920 atoms = [ 921 urllib.parse.unquote_to_bytes(x) 922 for x in QUOTED_SLASH_REGEX.split(path) 923 ] 924 except ValueError as ex: 925 self.simple_response('400 Bad Request', ex.args[0]) 926 return False 927 path = QUOTED_SLASH.join(atoms) 928 929 if not path.startswith(FORWARD_SLASH): 930 path = FORWARD_SLASH + path 931 932 if scheme is not EMPTY: 933 self.scheme = scheme 934 self.authority = authority 935 self.path = path 936 937 # Note that, like wsgiref and most other HTTP servers, 938 # we "% HEX HEX"-unquote the path but not the query string. 939 self.qs = qs 940 941 # Compare request and server HTTP protocol versions, in case our 942 # server does not support the requested protocol. Limit our output 943 # to min(req, server). We want the following output: 944 # request server actual written supported response 945 # protocol protocol response protocol feature set 946 # a 1.0 1.0 1.0 1.0 947 # b 1.0 1.1 1.1 1.0 948 # c 1.1 1.0 1.0 1.0 949 # d 1.1 1.1 1.1 1.1 950 # Notice that, in (b), the response will be "HTTP/1.1" even though 951 # the client only understands 1.0. RFC 2616 10.5.6 says we should 952 # only return 505 if the _major_ version is different. 953 sp = int(self.server.protocol[5]), int(self.server.protocol[7]) 954 955 if sp[0] != rp[0]: 956 self.simple_response('505 HTTP Version Not Supported') 957 return False 958 959 self.request_protocol = req_protocol 960 self.response_protocol = 'HTTP/%s.%s' % min(rp, sp) 961 962 return True 963 964 def read_request_headers(self): # noqa: C901 # FIXME 965 """Read ``self.rfile`` into ``self.inheaders``. 966 967 Ref: :py:attr:`self.inheaders <HTTPRequest.outheaders>`. 968 969 :returns: success status 970 :rtype: bool 971 """ 972 # then all the http headers 973 try: 974 self.header_reader(self.rfile, self.inheaders) 975 except ValueError as ex: 976 self.simple_response('400 Bad Request', ex.args[0]) 977 return False 978 979 mrbs = self.server.max_request_body_size 980 981 try: 982 cl = int(self.inheaders.get(b'Content-Length', 0)) 983 except ValueError: 984 self.simple_response( 985 '400 Bad Request', 986 'Malformed Content-Length Header.', 987 ) 988 return False 989 990 if mrbs and cl > mrbs: 991 self.simple_response( 992 '413 Request Entity Too Large', 993 'The entity sent with the request exceeds the maximum ' 994 'allowed bytes.', 995 ) 996 return False 997 998 # Persistent connection support 999 if self.response_protocol == 'HTTP/1.1': 1000 # Both server and client are HTTP/1.1 1001 if self.inheaders.get(b'Connection', b'') == b'close': 1002 self.close_connection = True 1003 else: 1004 # Either the server or client (or both) are HTTP/1.0 1005 if self.inheaders.get(b'Connection', b'') != b'Keep-Alive': 1006 self.close_connection = True 1007 1008 # Transfer-Encoding support 1009 te = None 1010 if self.response_protocol == 'HTTP/1.1': 1011 te = self.inheaders.get(b'Transfer-Encoding') 1012 if te: 1013 te = [x.strip().lower() for x in te.split(b',') if x.strip()] 1014 1015 self.chunked_read = False 1016 1017 if te: 1018 for enc in te: 1019 if enc == b'chunked': 1020 self.chunked_read = True 1021 else: 1022 # Note that, even if we see "chunked", we must reject 1023 # if there is an extension we don't recognize. 1024 self.simple_response('501 Unimplemented') 1025 self.close_connection = True 1026 return False 1027 1028 # From PEP 333: 1029 # "Servers and gateways that implement HTTP 1.1 must provide 1030 # transparent support for HTTP 1.1's "expect/continue" mechanism. 1031 # This may be done in any of several ways: 1032 # 1. Respond to requests containing an Expect: 100-continue request 1033 # with an immediate "100 Continue" response, and proceed normally. 1034 # 2. Proceed with the request normally, but provide the application 1035 # with a wsgi.input stream that will send the "100 Continue" 1036 # response if/when the application first attempts to read from 1037 # the input stream. The read request must then remain blocked 1038 # until the client responds. 1039 # 3. Wait until the client decides that the server does not support 1040 # expect/continue, and sends the request body on its own. 1041 # (This is suboptimal, and is not recommended.) 1042 # 1043 # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, 1044 # but it seems like it would be a big slowdown for such a rare case. 1045 if self.inheaders.get(b'Expect', b'') == b'100-continue': 1046 # Don't use simple_response here, because it emits headers 1047 # we don't want. See 1048 # https://github.com/cherrypy/cherrypy/issues/951 1049 msg = b''.join(( 1050 self.server.protocol.encode('ascii'), SPACE, b'100 Continue', 1051 CRLF, CRLF, 1052 )) 1053 try: 1054 self.conn.wfile.write(msg) 1055 except socket.error as ex: 1056 if ex.args[0] not in errors.socket_errors_to_ignore: 1057 raise 1058 return True 1059 1060 def respond(self): 1061 """Call the gateway and write its iterable output.""" 1062 mrbs = self.server.max_request_body_size 1063 if self.chunked_read: 1064 self.rfile = ChunkedRFile(self.conn.rfile, mrbs) 1065 else: 1066 cl = int(self.inheaders.get(b'Content-Length', 0)) 1067 if mrbs and mrbs < cl: 1068 if not self.sent_headers: 1069 self.simple_response( 1070 '413 Request Entity Too Large', 1071 'The entity sent with the request exceeds the ' 1072 'maximum allowed bytes.', 1073 ) 1074 return 1075 self.rfile = KnownLengthRFile(self.conn.rfile, cl) 1076 1077 self.server.gateway(self).respond() 1078 self.ready and self.ensure_headers_sent() 1079 1080 if self.chunked_write: 1081 self.conn.wfile.write(b'0\r\n\r\n') 1082 1083 def simple_response(self, status, msg=''): 1084 """Write a simple response back to the client.""" 1085 status = str(status) 1086 proto_status = '%s %s\r\n' % (self.server.protocol, status) 1087 content_length = 'Content-Length: %s\r\n' % len(msg) 1088 content_type = 'Content-Type: text/plain\r\n' 1089 buf = [ 1090 proto_status.encode('ISO-8859-1'), 1091 content_length.encode('ISO-8859-1'), 1092 content_type.encode('ISO-8859-1'), 1093 ] 1094 1095 if status[:3] in ('413', '414'): 1096 # Request Entity Too Large / Request-URI Too Long 1097 self.close_connection = True 1098 if self.response_protocol == 'HTTP/1.1': 1099 # This will not be true for 414, since read_request_line 1100 # usually raises 414 before reading the whole line, and we 1101 # therefore cannot know the proper response_protocol. 1102 buf.append(b'Connection: close\r\n') 1103 else: 1104 # HTTP/1.0 had no 413/414 status nor Connection header. 1105 # Emit 400 instead and trust the message body is enough. 1106 status = '400 Bad Request' 1107 1108 buf.append(CRLF) 1109 if msg: 1110 if isinstance(msg, six.text_type): 1111 msg = msg.encode('ISO-8859-1') 1112 buf.append(msg) 1113 1114 try: 1115 self.conn.wfile.write(EMPTY.join(buf)) 1116 except socket.error as ex: 1117 if ex.args[0] not in errors.socket_errors_to_ignore: 1118 raise 1119 1120 def ensure_headers_sent(self): 1121 """Ensure headers are sent to the client if not already sent.""" 1122 if not self.sent_headers: 1123 self.sent_headers = True 1124 self.send_headers() 1125 1126 def write(self, chunk): 1127 """Write unbuffered data to the client.""" 1128 if self.chunked_write and chunk: 1129 chunk_size_hex = hex(len(chunk))[2:].encode('ascii') 1130 buf = [chunk_size_hex, CRLF, chunk, CRLF] 1131 self.conn.wfile.write(EMPTY.join(buf)) 1132 else: 1133 self.conn.wfile.write(chunk) 1134 1135 def send_headers(self): # noqa: C901 # FIXME 1136 """Assert, process, and send the HTTP response message-headers. 1137 1138 You must set ``self.status``, and :py:attr:`self.outheaders 1139 <HTTPRequest.outheaders>` before calling this. 1140 """ 1141 hkeys = [key.lower() for key, value in self.outheaders] 1142 status = int(self.status[:3]) 1143 1144 if status == 413: 1145 # Request Entity Too Large. Close conn to avoid garbage. 1146 self.close_connection = True 1147 elif b'content-length' not in hkeys: 1148 # "All 1xx (informational), 204 (no content), 1149 # and 304 (not modified) responses MUST NOT 1150 # include a message-body." So no point chunking. 1151 if status < 200 or status in (204, 205, 304): 1152 pass 1153 else: 1154 needs_chunked = ( 1155 self.response_protocol == 'HTTP/1.1' 1156 and self.method != b'HEAD' 1157 ) 1158 if needs_chunked: 1159 # Use the chunked transfer-coding 1160 self.chunked_write = True 1161 self.outheaders.append((b'Transfer-Encoding', b'chunked')) 1162 else: 1163 # Closing the conn is the only way to determine len. 1164 self.close_connection = True 1165 1166 # Override the decision to not close the connection if the connection 1167 # manager doesn't have space for it. 1168 if not self.close_connection: 1169 can_keep = self.server.can_add_keepalive_connection 1170 self.close_connection = not can_keep 1171 1172 if b'connection' not in hkeys: 1173 if self.response_protocol == 'HTTP/1.1': 1174 # Both server and client are HTTP/1.1 or better 1175 if self.close_connection: 1176 self.outheaders.append((b'Connection', b'close')) 1177 else: 1178 # Server and/or client are HTTP/1.0 1179 if not self.close_connection: 1180 self.outheaders.append((b'Connection', b'Keep-Alive')) 1181 1182 if (b'Connection', b'Keep-Alive') in self.outheaders: 1183 self.outheaders.append(( 1184 b'Keep-Alive', 1185 u'timeout={connection_timeout}'. 1186 format(connection_timeout=self.server.timeout). 1187 encode('ISO-8859-1'), 1188 )) 1189 1190 if (not self.close_connection) and (not self.chunked_read): 1191 # Read any remaining request body data on the socket. 1192 # "If an origin server receives a request that does not include an 1193 # Expect request-header field with the "100-continue" expectation, 1194 # the request includes a request body, and the server responds 1195 # with a final status code before reading the entire request body 1196 # from the transport connection, then the server SHOULD NOT close 1197 # the transport connection until it has read the entire request, 1198 # or until the client closes the connection. Otherwise, the client 1199 # might not reliably receive the response message. However, this 1200 # requirement is not be construed as preventing a server from 1201 # defending itself against denial-of-service attacks, or from 1202 # badly broken client implementations." 1203 remaining = getattr(self.rfile, 'remaining', 0) 1204 if remaining > 0: 1205 self.rfile.read(remaining) 1206 1207 if b'date' not in hkeys: 1208 self.outheaders.append(( 1209 b'Date', 1210 email.utils.formatdate(usegmt=True).encode('ISO-8859-1'), 1211 )) 1212 1213 if b'server' not in hkeys: 1214 self.outheaders.append(( 1215 b'Server', 1216 self.server.server_name.encode('ISO-8859-1'), 1217 )) 1218 1219 proto = self.server.protocol.encode('ascii') 1220 buf = [proto + SPACE + self.status + CRLF] 1221 for k, v in self.outheaders: 1222 buf.append(k + COLON + SPACE + v + CRLF) 1223 buf.append(CRLF) 1224 self.conn.wfile.write(EMPTY.join(buf)) 1225 1226 1227class HTTPConnection: 1228 """An HTTP connection (active socket).""" 1229 1230 remote_addr = None 1231 remote_port = None 1232 ssl_env = None 1233 rbufsize = io.DEFAULT_BUFFER_SIZE 1234 wbufsize = io.DEFAULT_BUFFER_SIZE 1235 RequestHandlerClass = HTTPRequest 1236 peercreds_enabled = False 1237 peercreds_resolve_enabled = False 1238 1239 # Fields set by ConnectionManager. 1240 last_used = None 1241 1242 def __init__(self, server, sock, makefile=MakeFile): 1243 """Initialize HTTPConnection instance. 1244 1245 Args: 1246 server (HTTPServer): web server object receiving this request 1247 sock (socket._socketobject): the raw socket object (usually 1248 TCP) for this connection 1249 makefile (file): a fileobject class for reading from the socket 1250 """ 1251 self.server = server 1252 self.socket = sock 1253 self.rfile = makefile(sock, 'rb', self.rbufsize) 1254 self.wfile = makefile(sock, 'wb', self.wbufsize) 1255 self.requests_seen = 0 1256 1257 self.peercreds_enabled = self.server.peercreds_enabled 1258 self.peercreds_resolve_enabled = self.server.peercreds_resolve_enabled 1259 1260 # LRU cached methods: 1261 # Ref: https://stackoverflow.com/a/14946506/595220 1262 self.resolve_peer_creds = ( 1263 lru_cache(maxsize=1)(self.resolve_peer_creds) 1264 ) 1265 self.get_peer_creds = ( 1266 lru_cache(maxsize=1)(self.get_peer_creds) 1267 ) 1268 1269 def communicate(self): # noqa: C901 # FIXME 1270 """Read each request and respond appropriately. 1271 1272 Returns true if the connection should be kept open. 1273 """ 1274 request_seen = False 1275 try: 1276 req = self.RequestHandlerClass(self.server, self) 1277 req.parse_request() 1278 if self.server.stats['Enabled']: 1279 self.requests_seen += 1 1280 if not req.ready: 1281 # Something went wrong in the parsing (and the server has 1282 # probably already made a simple_response). Return and 1283 # let the conn close. 1284 return False 1285 1286 request_seen = True 1287 req.respond() 1288 if not req.close_connection: 1289 return True 1290 except socket.error as ex: 1291 errnum = ex.args[0] 1292 # sadly SSL sockets return a different (longer) time out string 1293 timeout_errs = 'timed out', 'The read operation timed out' 1294 if errnum in timeout_errs: 1295 # Don't error if we're between requests; only error 1296 # if 1) no request has been started at all, or 2) we're 1297 # in the middle of a request. 1298 # See https://github.com/cherrypy/cherrypy/issues/853 1299 if (not request_seen) or (req and req.started_request): 1300 self._conditional_error(req, '408 Request Timeout') 1301 elif errnum not in errors.socket_errors_to_ignore: 1302 self.server.error_log( 1303 'socket.error %s' % repr(errnum), 1304 level=logging.WARNING, traceback=True, 1305 ) 1306 self._conditional_error(req, '500 Internal Server Error') 1307 except (KeyboardInterrupt, SystemExit): 1308 raise 1309 except errors.FatalSSLAlert: 1310 pass 1311 except errors.NoSSLError: 1312 self._handle_no_ssl(req) 1313 except Exception as ex: 1314 self.server.error_log( 1315 repr(ex), level=logging.ERROR, traceback=True, 1316 ) 1317 self._conditional_error(req, '500 Internal Server Error') 1318 return False 1319 1320 linger = False 1321 1322 def _handle_no_ssl(self, req): 1323 if not req or req.sent_headers: 1324 return 1325 # Unwrap wfile 1326 try: 1327 resp_sock = self.socket._sock 1328 except AttributeError: 1329 # self.socket is of OpenSSL.SSL.Connection type 1330 resp_sock = self.socket._socket 1331 self.wfile = StreamWriter(resp_sock, 'wb', self.wbufsize) 1332 msg = ( 1333 'The client sent a plain HTTP request, but ' 1334 'this server only speaks HTTPS on this port.' 1335 ) 1336 req.simple_response('400 Bad Request', msg) 1337 self.linger = True 1338 1339 def _conditional_error(self, req, response): 1340 """Respond with an error. 1341 1342 Don't bother writing if a response 1343 has already started being written. 1344 """ 1345 if not req or req.sent_headers: 1346 return 1347 1348 try: 1349 req.simple_response(response) 1350 except errors.FatalSSLAlert: 1351 pass 1352 except errors.NoSSLError: 1353 self._handle_no_ssl(req) 1354 1355 def close(self): 1356 """Close the socket underlying this connection.""" 1357 self.rfile.close() 1358 1359 if not self.linger: 1360 self._close_kernel_socket() 1361 # close the socket file descriptor 1362 # (will be closed in the OS if there is no 1363 # other reference to the underlying socket) 1364 self.socket.close() 1365 else: 1366 # On the other hand, sometimes we want to hang around for a bit 1367 # to make sure the client has a chance to read our entire 1368 # response. Skipping the close() calls here delays the FIN 1369 # packet until the socket object is garbage-collected later. 1370 # Someday, perhaps, we'll do the full lingering_close that 1371 # Apache does, but not today. 1372 pass 1373 1374 def get_peer_creds(self): # LRU cached on per-instance basis, see __init__ 1375 """Return the PID/UID/GID tuple of the peer socket for UNIX sockets. 1376 1377 This function uses SO_PEERCRED to query the UNIX PID, UID, GID 1378 of the peer, which is only available if the bind address is 1379 a UNIX domain socket. 1380 1381 Raises: 1382 NotImplementedError: in case of unsupported socket type 1383 RuntimeError: in case of SO_PEERCRED lookup unsupported or disabled 1384 1385 """ 1386 PEERCRED_STRUCT_DEF = '3i' 1387 1388 if IS_WINDOWS or self.socket.family != socket.AF_UNIX: 1389 raise NotImplementedError( 1390 'SO_PEERCRED is only supported in Linux kernel and WSL', 1391 ) 1392 elif not self.peercreds_enabled: 1393 raise RuntimeError( 1394 'Peer creds lookup is disabled within this server', 1395 ) 1396 1397 try: 1398 peer_creds = self.socket.getsockopt( 1399 # FIXME: Use LOCAL_CREDS for BSD-like OSs 1400 # Ref: https://gist.github.com/LucaFilipozzi/e4f1e118202aff27af6aadebda1b5d91 # noqa 1401 socket.SOL_SOCKET, socket.SO_PEERCRED, 1402 struct.calcsize(PEERCRED_STRUCT_DEF), 1403 ) 1404 except socket.error as socket_err: 1405 """Non-Linux kernels don't support SO_PEERCRED. 1406 1407 Refs: 1408 http://welz.org.za/notes/on-peer-cred.html 1409 https://github.com/daveti/tcpSockHack 1410 msdn.microsoft.com/en-us/commandline/wsl/release_notes#build-15025 1411 """ 1412 six.raise_from( # 3.6+: raise RuntimeError from socket_err 1413 RuntimeError, 1414 socket_err, 1415 ) 1416 else: 1417 pid, uid, gid = struct.unpack(PEERCRED_STRUCT_DEF, peer_creds) 1418 return pid, uid, gid 1419 1420 @property 1421 def peer_pid(self): 1422 """Return the id of the connected peer process.""" 1423 pid, _, _ = self.get_peer_creds() 1424 return pid 1425 1426 @property 1427 def peer_uid(self): 1428 """Return the user id of the connected peer process.""" 1429 _, uid, _ = self.get_peer_creds() 1430 return uid 1431 1432 @property 1433 def peer_gid(self): 1434 """Return the group id of the connected peer process.""" 1435 _, _, gid = self.get_peer_creds() 1436 return gid 1437 1438 def resolve_peer_creds(self): # LRU cached on per-instance basis 1439 """Look up the username and group tuple of the ``PEERCREDS``. 1440 1441 :returns: the username and group tuple of the ``PEERCREDS`` 1442 1443 :raises NotImplementedError: if the OS is unsupported 1444 :raises RuntimeError: if UID/GID lookup is unsupported or disabled 1445 """ 1446 if not IS_UID_GID_RESOLVABLE: 1447 raise NotImplementedError( 1448 'UID/GID lookup is unavailable under current platform. ' 1449 'It can only be done under UNIX-like OS ' 1450 'but not under the Google App Engine', 1451 ) 1452 elif not self.peercreds_resolve_enabled: 1453 raise RuntimeError( 1454 'UID/GID lookup is disabled within this server', 1455 ) 1456 1457 user = pwd.getpwuid(self.peer_uid).pw_name # [0] 1458 group = grp.getgrgid(self.peer_gid).gr_name # [0] 1459 1460 return user, group 1461 1462 @property 1463 def peer_user(self): 1464 """Return the username of the connected peer process.""" 1465 user, _ = self.resolve_peer_creds() 1466 return user 1467 1468 @property 1469 def peer_group(self): 1470 """Return the group of the connected peer process.""" 1471 _, group = self.resolve_peer_creds() 1472 return group 1473 1474 def _close_kernel_socket(self): 1475 """Terminate the connection at the transport level.""" 1476 # Honor ``sock_shutdown`` for PyOpenSSL connections. 1477 shutdown = getattr( 1478 self.socket, 'sock_shutdown', 1479 self.socket.shutdown, 1480 ) 1481 1482 try: 1483 shutdown(socket.SHUT_RDWR) # actually send a TCP FIN 1484 except errors.acceptable_sock_shutdown_exceptions: 1485 pass 1486 except socket.error as e: 1487 if e.errno not in errors.acceptable_sock_shutdown_error_codes: 1488 raise 1489 1490 1491class HTTPServer: 1492 """An HTTP server.""" 1493 1494 _bind_addr = '127.0.0.1' 1495 _interrupt = None 1496 1497 gateway = None 1498 """A Gateway instance.""" 1499 1500 minthreads = None 1501 """The minimum number of worker threads to create (default 10).""" 1502 1503 maxthreads = None 1504 """The maximum number of worker threads to create. 1505 1506 (default -1 = no limit)""" 1507 1508 server_name = None 1509 """The name of the server; defaults to ``self.version``.""" 1510 1511 protocol = 'HTTP/1.1' 1512 """The version string to write in the Status-Line of all HTTP responses. 1513 1514 For example, "HTTP/1.1" is the default. This also limits the supported 1515 features used in the response.""" 1516 1517 request_queue_size = 5 1518 """The 'backlog' arg to socket.listen(); max queued connections. 1519 1520 (default 5).""" 1521 1522 shutdown_timeout = 5 1523 """The total time to wait for worker threads to cleanly exit. 1524 1525 Specified in seconds.""" 1526 1527 timeout = 10 1528 """The timeout in seconds for accepted connections (default 10).""" 1529 1530 expiration_interval = 0.5 1531 """The interval, in seconds, at which the server checks for 1532 expired connections (default 0.5). 1533 """ 1534 1535 version = 'Cheroot/{version!s}'.format(version=__version__) 1536 """A version string for the HTTPServer.""" 1537 1538 software = None 1539 """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. 1540 1541 If None, this defaults to ``'%s Server' % self.version``. 1542 """ 1543 1544 ready = False 1545 """Internal flag which indicating the socket is accepting connections.""" 1546 1547 max_request_header_size = 0 1548 """The maximum size, in bytes, for request headers, or 0 for no limit.""" 1549 1550 max_request_body_size = 0 1551 """The maximum size, in bytes, for request bodies, or 0 for no limit.""" 1552 1553 nodelay = True 1554 """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" 1555 1556 ConnectionClass = HTTPConnection 1557 """The class to use for handling HTTP connections.""" 1558 1559 ssl_adapter = None 1560 """An instance of ``ssl.Adapter`` (or a subclass). 1561 1562 Ref: :py:class:`ssl.Adapter <cheroot.ssl.Adapter>`. 1563 1564 You must have the corresponding TLS driver library installed. 1565 """ 1566 1567 peercreds_enabled = False 1568 """ 1569 If :py:data:`True`, peer creds will be looked up via UNIX domain socket. 1570 """ 1571 1572 peercreds_resolve_enabled = False 1573 """ 1574 If :py:data:`True`, username/group will be looked up in the OS from 1575 ``PEERCREDS``-provided IDs. 1576 """ 1577 1578 keep_alive_conn_limit = 10 1579 """The maximum number of waiting keep-alive connections that will be kept open. 1580 1581 Default is 10. Set to None to have unlimited connections.""" 1582 1583 def __init__( 1584 self, bind_addr, gateway, 1585 minthreads=10, maxthreads=-1, server_name=None, 1586 peercreds_enabled=False, peercreds_resolve_enabled=False, 1587 ): 1588 """Initialize HTTPServer instance. 1589 1590 Args: 1591 bind_addr (tuple): network interface to listen to 1592 gateway (Gateway): gateway for processing HTTP requests 1593 minthreads (int): minimum number of threads for HTTP thread pool 1594 maxthreads (int): maximum number of threads for HTTP thread pool 1595 server_name (str): web server name to be advertised via Server 1596 HTTP header 1597 """ 1598 self.bind_addr = bind_addr 1599 self.gateway = gateway 1600 1601 self.requests = threadpool.ThreadPool( 1602 self, min=minthreads or 1, max=maxthreads, 1603 ) 1604 1605 if not server_name: 1606 server_name = self.version 1607 self.server_name = server_name 1608 self.peercreds_enabled = peercreds_enabled 1609 self.peercreds_resolve_enabled = ( 1610 peercreds_resolve_enabled and peercreds_enabled 1611 ) 1612 self.clear_stats() 1613 1614 def clear_stats(self): 1615 """Reset server stat counters..""" 1616 self._start_time = None 1617 self._run_time = 0 1618 self.stats = { 1619 'Enabled': False, 1620 'Bind Address': lambda s: repr(self.bind_addr), 1621 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), 1622 'Accepts': 0, 1623 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), 1624 'Queue': lambda s: getattr(self.requests, 'qsize', None), 1625 'Threads': lambda s: len(getattr(self.requests, '_threads', [])), 1626 'Threads Idle': lambda s: getattr(self.requests, 'idle', None), 1627 'Socket Errors': 0, 1628 'Requests': lambda s: (not s['Enabled']) and -1 or sum( 1629 (w['Requests'](w) for w in s['Worker Threads'].values()), 0, 1630 ), 1631 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( 1632 (w['Bytes Read'](w) for w in s['Worker Threads'].values()), 0, 1633 ), 1634 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( 1635 (w['Bytes Written'](w) for w in s['Worker Threads'].values()), 1636 0, 1637 ), 1638 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( 1639 (w['Work Time'](w) for w in s['Worker Threads'].values()), 0, 1640 ), 1641 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( 1642 ( 1643 w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) 1644 for w in s['Worker Threads'].values() 1645 ), 0, 1646 ), 1647 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( 1648 ( 1649 w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) 1650 for w in s['Worker Threads'].values() 1651 ), 0, 1652 ), 1653 'Worker Threads': {}, 1654 } 1655 logging.statistics['Cheroot HTTPServer %d' % id(self)] = self.stats 1656 1657 def runtime(self): 1658 """Return server uptime.""" 1659 if self._start_time is None: 1660 return self._run_time 1661 else: 1662 return self._run_time + (time.time() - self._start_time) 1663 1664 def __str__(self): 1665 """Render Server instance representing bind address.""" 1666 return '%s.%s(%r)' % ( 1667 self.__module__, self.__class__.__name__, 1668 self.bind_addr, 1669 ) 1670 1671 @property 1672 def bind_addr(self): 1673 """Return the interface on which to listen for connections. 1674 1675 For TCP sockets, a (host, port) tuple. Host values may be any 1676 :term:`IPv4` or :term:`IPv6` address, or any valid hostname. 1677 The string 'localhost' is a synonym for '127.0.0.1' (or '::1', 1678 if your hosts file prefers :term:`IPv6`). 1679 The string '0.0.0.0' is a special :term:`IPv4` entry meaning 1680 "any active interface" (INADDR_ANY), and '::' is the similar 1681 IN6ADDR_ANY for :term:`IPv6`. 1682 The empty string or :py:data:`None` are not allowed. 1683 1684 For UNIX sockets, supply the file name as a string. 1685 1686 Systemd socket activation is automatic and doesn't require tempering 1687 with this variable. 1688 1689 .. glossary:: 1690 1691 :abbr:`IPv4 (Internet Protocol version 4)` 1692 Internet Protocol version 4 1693 1694 :abbr:`IPv6 (Internet Protocol version 6)` 1695 Internet Protocol version 6 1696 """ 1697 return self._bind_addr 1698 1699 @bind_addr.setter 1700 def bind_addr(self, value): 1701 """Set the interface on which to listen for connections.""" 1702 if isinstance(value, tuple) and value[0] in ('', None): 1703 # Despite the socket module docs, using '' does not 1704 # allow AI_PASSIVE to work. Passing None instead 1705 # returns '0.0.0.0' like we want. In other words: 1706 # host AI_PASSIVE result 1707 # '' Y 192.168.x.y 1708 # '' N 192.168.x.y 1709 # None Y 0.0.0.0 1710 # None N 127.0.0.1 1711 # But since you can get the same effect with an explicit 1712 # '0.0.0.0', we deny both the empty string and None as values. 1713 raise ValueError( 1714 "Host values of '' or None are not allowed. " 1715 "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " 1716 'to listen on all active interfaces.', 1717 ) 1718 self._bind_addr = value 1719 1720 def safe_start(self): 1721 """Run the server forever, and stop it cleanly on exit.""" 1722 try: 1723 self.start() 1724 except (KeyboardInterrupt, IOError): 1725 # The time.sleep call might raise 1726 # "IOError: [Errno 4] Interrupted function call" on KBInt. 1727 self.error_log('Keyboard Interrupt: shutting down') 1728 self.stop() 1729 raise 1730 except SystemExit: 1731 self.error_log('SystemExit raised: shutting down') 1732 self.stop() 1733 raise 1734 1735 def prepare(self): # noqa: C901 # FIXME 1736 """Prepare server to serving requests. 1737 1738 It binds a socket's port, setups the socket to ``listen()`` and does 1739 other preparing things. 1740 """ 1741 self._interrupt = None 1742 1743 if self.software is None: 1744 self.software = '%s Server' % self.version 1745 1746 # Select the appropriate socket 1747 self.socket = None 1748 msg = 'No socket could be created' 1749 if os.getenv('LISTEN_PID', None): 1750 # systemd socket activation 1751 self.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM) 1752 elif isinstance(self.bind_addr, (six.text_type, six.binary_type)): 1753 # AF_UNIX socket 1754 try: 1755 self.bind_unix_socket(self.bind_addr) 1756 except socket.error as serr: 1757 msg = '%s -- (%s: %s)' % (msg, self.bind_addr, serr) 1758 six.raise_from(socket.error(msg), serr) 1759 else: 1760 # AF_INET or AF_INET6 socket 1761 # Get the correct address family for our host (allows IPv6 1762 # addresses) 1763 host, port = self.bind_addr 1764 try: 1765 info = socket.getaddrinfo( 1766 host, port, socket.AF_UNSPEC, 1767 socket.SOCK_STREAM, 0, socket.AI_PASSIVE, 1768 ) 1769 except socket.gaierror: 1770 sock_type = socket.AF_INET 1771 bind_addr = self.bind_addr 1772 1773 if ':' in host: 1774 sock_type = socket.AF_INET6 1775 bind_addr = bind_addr + (0, 0) 1776 1777 info = [(sock_type, socket.SOCK_STREAM, 0, '', bind_addr)] 1778 1779 for res in info: 1780 af, socktype, proto, canonname, sa = res 1781 try: 1782 self.bind(af, socktype, proto) 1783 break 1784 except socket.error as serr: 1785 msg = '%s -- (%s: %s)' % (msg, sa, serr) 1786 if self.socket: 1787 self.socket.close() 1788 self.socket = None 1789 1790 if not self.socket: 1791 raise socket.error(msg) 1792 1793 # Timeout so KeyboardInterrupt can be caught on Win32 1794 self.socket.settimeout(1) 1795 self.socket.listen(self.request_queue_size) 1796 1797 # must not be accessed once stop() has been called 1798 self._connections = connections.ConnectionManager(self) 1799 1800 # Create worker threads 1801 self.requests.start() 1802 1803 self.ready = True 1804 self._start_time = time.time() 1805 1806 def serve(self): 1807 """Serve requests, after invoking :func:`prepare()`.""" 1808 while self.ready and not self.interrupt: 1809 try: 1810 self._connections.run(self.expiration_interval) 1811 except (KeyboardInterrupt, SystemExit): 1812 raise 1813 except Exception: 1814 self.error_log( 1815 'Error in HTTPServer.serve', level=logging.ERROR, 1816 traceback=True, 1817 ) 1818 1819 # raise exceptions reported by any worker threads, 1820 # such that the exception is raised from the serve() thread. 1821 if self.interrupt: 1822 while self._stopping_for_interrupt: 1823 time.sleep(0.1) 1824 if self.interrupt: 1825 raise self.interrupt 1826 1827 def start(self): 1828 """Run the server forever. 1829 1830 It is shortcut for invoking :func:`prepare()` then :func:`serve()`. 1831 """ 1832 # We don't have to trap KeyboardInterrupt or SystemExit here, 1833 # because cherrypy.server already does so, calling self.stop() for us. 1834 # If you're using this server with another framework, you should 1835 # trap those exceptions in whatever code block calls start(). 1836 self.prepare() 1837 self.serve() 1838 1839 @contextlib.contextmanager 1840 def _run_in_thread(self): 1841 """Context manager for running this server in a thread.""" 1842 self.prepare() 1843 thread = threading.Thread(target=self.serve) 1844 thread.setDaemon(True) 1845 thread.start() 1846 try: 1847 yield thread 1848 finally: 1849 self.stop() 1850 1851 @property 1852 def can_add_keepalive_connection(self): 1853 """Flag whether it is allowed to add a new keep-alive connection.""" 1854 return self.ready and self._connections.can_add_keepalive_connection 1855 1856 def put_conn(self, conn): 1857 """Put an idle connection back into the ConnectionManager.""" 1858 if self.ready: 1859 self._connections.put(conn) 1860 else: 1861 # server is shutting down, just close it 1862 conn.close() 1863 1864 def error_log(self, msg='', level=20, traceback=False): 1865 """Write error message to log. 1866 1867 Args: 1868 msg (str): error message 1869 level (int): logging level 1870 traceback (bool): add traceback to output or not 1871 """ 1872 # Override this in subclasses as desired 1873 sys.stderr.write('{msg!s}\n'.format(msg=msg)) 1874 sys.stderr.flush() 1875 if traceback: 1876 tblines = traceback_.format_exc() 1877 sys.stderr.write(tblines) 1878 sys.stderr.flush() 1879 1880 def bind(self, family, type, proto=0): 1881 """Create (or recreate) the actual socket object.""" 1882 sock = self.prepare_socket( 1883 self.bind_addr, 1884 family, type, proto, 1885 self.nodelay, self.ssl_adapter, 1886 ) 1887 sock = self.socket = self.bind_socket(sock, self.bind_addr) 1888 self.bind_addr = self.resolve_real_bind_addr(sock) 1889 return sock 1890 1891 def bind_unix_socket(self, bind_addr): # noqa: C901 # FIXME 1892 """Create (or recreate) a UNIX socket object.""" 1893 if IS_WINDOWS: 1894 """ 1895 Trying to access socket.AF_UNIX under Windows 1896 causes an AttributeError. 1897 """ 1898 raise ValueError( # or RuntimeError? 1899 'AF_UNIX sockets are not supported under Windows.', 1900 ) 1901 1902 fs_permissions = 0o777 # TODO: allow changing mode 1903 1904 try: 1905 # Make possible reusing the socket... 1906 os.unlink(self.bind_addr) 1907 except OSError: 1908 """ 1909 File does not exist, which is the primary goal anyway. 1910 """ 1911 except TypeError as typ_err: 1912 err_msg = str(typ_err) 1913 if ( 1914 'remove() argument 1 must be encoded ' 1915 'string without null bytes, not unicode' 1916 not in err_msg 1917 and 'embedded NUL character' not in err_msg # py34 1918 and 'argument must be a ' 1919 'string without NUL characters' not in err_msg # pypy2 1920 ): 1921 raise 1922 except ValueError as val_err: 1923 err_msg = str(val_err) 1924 if ( 1925 'unlink: embedded null ' 1926 'character in path' not in err_msg 1927 and 'embedded null byte' not in err_msg 1928 and 'argument must be a ' 1929 'string without NUL characters' not in err_msg # pypy3 1930 ): 1931 raise 1932 1933 sock = self.prepare_socket( 1934 bind_addr=bind_addr, 1935 family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, 1936 nodelay=self.nodelay, ssl_adapter=self.ssl_adapter, 1937 ) 1938 1939 try: 1940 """Linux way of pre-populating fs mode permissions.""" 1941 # Allow everyone access the socket... 1942 os.fchmod(sock.fileno(), fs_permissions) 1943 FS_PERMS_SET = True 1944 except OSError: 1945 FS_PERMS_SET = False 1946 1947 try: 1948 sock = self.bind_socket(sock, bind_addr) 1949 except socket.error: 1950 sock.close() 1951 raise 1952 1953 bind_addr = self.resolve_real_bind_addr(sock) 1954 1955 try: 1956 """FreeBSD/macOS pre-populating fs mode permissions.""" 1957 if not FS_PERMS_SET: 1958 try: 1959 os.lchmod(bind_addr, fs_permissions) 1960 except AttributeError: 1961 os.chmod(bind_addr, fs_permissions, follow_symlinks=False) 1962 FS_PERMS_SET = True 1963 except OSError: 1964 pass 1965 1966 if not FS_PERMS_SET: 1967 self.error_log( 1968 'Failed to set socket fs mode permissions', 1969 level=logging.WARNING, 1970 ) 1971 1972 self.bind_addr = bind_addr 1973 self.socket = sock 1974 return sock 1975 1976 @staticmethod 1977 def prepare_socket(bind_addr, family, type, proto, nodelay, ssl_adapter): 1978 """Create and prepare the socket object.""" 1979 sock = socket.socket(family, type, proto) 1980 connections.prevent_socket_inheritance(sock) 1981 1982 host, port = bind_addr[:2] 1983 IS_EPHEMERAL_PORT = port == 0 1984 1985 if not (IS_WINDOWS or IS_EPHEMERAL_PORT): 1986 """Enable SO_REUSEADDR for the current socket. 1987 1988 Skip for Windows (has different semantics) 1989 or ephemeral ports (can steal ports from others). 1990 1991 Refs: 1992 * https://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx 1993 * https://github.com/cherrypy/cheroot/issues/114 1994 * https://gavv.github.io/blog/ephemeral-port-reuse/ 1995 """ 1996 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 1997 if nodelay and not isinstance( 1998 bind_addr, 1999 (six.text_type, six.binary_type), 2000 ): 2001 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 2002 2003 if ssl_adapter is not None: 2004 sock = ssl_adapter.bind(sock) 2005 2006 # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), 2007 # activate dual-stack. See 2008 # https://github.com/cherrypy/cherrypy/issues/871. 2009 listening_ipv6 = ( 2010 hasattr(socket, 'AF_INET6') 2011 and family == socket.AF_INET6 2012 and host in ('::', '::0', '::0.0.0.0') 2013 ) 2014 if listening_ipv6: 2015 try: 2016 sock.setsockopt( 2017 socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0, 2018 ) 2019 except (AttributeError, socket.error): 2020 # Apparently, the socket option is not available in 2021 # this machine's TCP stack 2022 pass 2023 2024 return sock 2025 2026 @staticmethod 2027 def bind_socket(socket_, bind_addr): 2028 """Bind the socket to given interface.""" 2029 socket_.bind(bind_addr) 2030 return socket_ 2031 2032 @staticmethod 2033 def resolve_real_bind_addr(socket_): 2034 """Retrieve actual bind address from bound socket.""" 2035 # FIXME: keep requested bind_addr separate real bound_addr (port 2036 # is different in case of ephemeral port 0) 2037 bind_addr = socket_.getsockname() 2038 if socket_.family in ( 2039 # Windows doesn't have socket.AF_UNIX, so not using it in check 2040 socket.AF_INET, 2041 socket.AF_INET6, 2042 ): 2043 """UNIX domain sockets are strings or bytes. 2044 2045 In case of bytes with a leading null-byte it's an abstract socket. 2046 """ 2047 return bind_addr[:2] 2048 2049 if isinstance(bind_addr, six.binary_type): 2050 bind_addr = bton(bind_addr) 2051 2052 return bind_addr 2053 2054 def process_conn(self, conn): 2055 """Process an incoming HTTPConnection.""" 2056 try: 2057 self.requests.put(conn) 2058 except queue.Full: 2059 # Just drop the conn. TODO: write 503 back? 2060 conn.close() 2061 2062 @property 2063 def interrupt(self): 2064 """Flag interrupt of the server.""" 2065 return self._interrupt 2066 2067 @property 2068 def _stopping_for_interrupt(self): 2069 """Return whether the server is responding to an interrupt.""" 2070 return self._interrupt is _STOPPING_FOR_INTERRUPT 2071 2072 @interrupt.setter 2073 def interrupt(self, interrupt): 2074 """Perform the shutdown of this server and save the exception. 2075 2076 Typically invoked by a worker thread in 2077 :py:mod:`~cheroot.workers.threadpool`, the exception is raised 2078 from the thread running :py:meth:`serve` once :py:meth:`stop` 2079 has completed. 2080 """ 2081 self._interrupt = _STOPPING_FOR_INTERRUPT 2082 self.stop() 2083 self._interrupt = interrupt 2084 2085 def stop(self): # noqa: C901 # FIXME 2086 """Gracefully shutdown a server that is serving forever.""" 2087 if not self.ready: 2088 return # already stopped 2089 2090 self.ready = False 2091 if self._start_time is not None: 2092 self._run_time += (time.time() - self._start_time) 2093 self._start_time = None 2094 2095 self._connections.stop() 2096 2097 sock = getattr(self, 'socket', None) 2098 if sock: 2099 if not isinstance( 2100 self.bind_addr, 2101 (six.text_type, six.binary_type), 2102 ): 2103 # Touch our own socket to make accept() return immediately. 2104 try: 2105 host, port = sock.getsockname()[:2] 2106 except socket.error as ex: 2107 if ex.args[0] not in errors.socket_errors_to_ignore: 2108 # Changed to use error code and not message 2109 # See 2110 # https://github.com/cherrypy/cherrypy/issues/860. 2111 raise 2112 else: 2113 # Note that we're explicitly NOT using AI_PASSIVE, 2114 # here, because we want an actual IP to touch. 2115 # localhost won't work if we've bound to a public IP, 2116 # but it will if we bound to '0.0.0.0' (INADDR_ANY). 2117 for res in socket.getaddrinfo( 2118 host, port, socket.AF_UNSPEC, 2119 socket.SOCK_STREAM, 2120 ): 2121 af, socktype, proto, canonname, sa = res 2122 s = None 2123 try: 2124 s = socket.socket(af, socktype, proto) 2125 # See 2126 # https://groups.google.com/group/cherrypy-users/ 2127 # browse_frm/thread/bbfe5eb39c904fe0 2128 s.settimeout(1.0) 2129 s.connect((host, port)) 2130 s.close() 2131 except socket.error: 2132 if s: 2133 s.close() 2134 if hasattr(sock, 'close'): 2135 sock.close() 2136 self.socket = None 2137 2138 self._connections.close() 2139 self.requests.stop(self.shutdown_timeout) 2140 2141 2142class Gateway: 2143 """Base class to interface HTTPServer with other systems, such as WSGI.""" 2144 2145 def __init__(self, req): 2146 """Initialize Gateway instance with request. 2147 2148 Args: 2149 req (HTTPRequest): current HTTP request 2150 """ 2151 self.req = req 2152 2153 def respond(self): 2154 """Process the current request. Must be overridden in a subclass.""" 2155 raise NotImplementedError # pragma: no cover 2156 2157 2158# These may either be ssl.Adapter subclasses or the string names 2159# of such classes (in which case they will be lazily loaded). 2160ssl_adapters = { 2161 'builtin': 'cheroot.ssl.builtin.BuiltinSSLAdapter', 2162 'pyopenssl': 'cheroot.ssl.pyopenssl.pyOpenSSLAdapter', 2163} 2164 2165 2166def get_ssl_adapter_class(name='builtin'): 2167 """Return an SSL adapter class for the given name.""" 2168 adapter = ssl_adapters[name.lower()] 2169 if isinstance(adapter, six.string_types): 2170 last_dot = adapter.rfind('.') 2171 attr_name = adapter[last_dot + 1:] 2172 mod_path = adapter[:last_dot] 2173 2174 try: 2175 mod = sys.modules[mod_path] 2176 if mod is None: 2177 raise KeyError() 2178 except KeyError: 2179 # The last [''] is important. 2180 mod = __import__(mod_path, globals(), locals(), ['']) 2181 2182 # Let an AttributeError propagate outward. 2183 try: 2184 adapter = getattr(mod, attr_name) 2185 except AttributeError: 2186 raise AttributeError( 2187 "'%s' object has no attribute '%s'" 2188 % (mod_path, attr_name), 2189 ) 2190 2191 return adapter 2192