1# Copyright 2014-2016 OpenMarket Ltd 2# Copyright 2018 New Vector Ltd 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import abc 17import html 18import logging 19import types 20import urllib 21from http import HTTPStatus 22from inspect import isawaitable 23from typing import ( 24 TYPE_CHECKING, 25 Any, 26 Awaitable, 27 Callable, 28 Dict, 29 Iterable, 30 Iterator, 31 List, 32 NoReturn, 33 Optional, 34 Pattern, 35 Tuple, 36 Union, 37) 38 39import attr 40import jinja2 41from canonicaljson import encode_canonical_json 42from typing_extensions import Protocol 43from zope.interface import implementer 44 45from twisted.internet import defer, interfaces 46from twisted.python import failure 47from twisted.web import resource 48from twisted.web.server import NOT_DONE_YET, Request 49from twisted.web.static import File 50from twisted.web.util import redirectTo 51 52from synapse.api.errors import ( 53 CodeMessageException, 54 Codes, 55 RedirectException, 56 SynapseError, 57 UnrecognizedRequestError, 58) 59from synapse.http.site import SynapseRequest 60from synapse.logging.context import defer_to_thread, preserve_fn, run_in_background 61from synapse.logging.opentracing import active_span, start_active_span, trace_servlet 62from synapse.util import json_encoder 63from synapse.util.caches import intern_dict 64from synapse.util.iterutils import chunk_seq 65 66if TYPE_CHECKING: 67 import opentracing 68 69 from synapse.server import HomeServer 70 71logger = logging.getLogger(__name__) 72 73HTML_ERROR_TEMPLATE = """<!DOCTYPE html> 74<html lang=en> 75 <head> 76 <meta charset="utf-8"> 77 <title>Error {code}</title> 78 </head> 79 <body> 80 <p>{msg}</p> 81 </body> 82</html> 83""" 84 85 86def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: 87 """Sends a JSON error response to clients.""" 88 89 if f.check(SynapseError): 90 # mypy doesn't understand that f.check asserts the type. 91 exc: SynapseError = f.value # type: ignore 92 error_code = exc.code 93 error_dict = exc.error_dict() 94 95 logger.info("%s SynapseError: %s - %s", request, error_code, exc.msg) 96 else: 97 error_code = 500 98 error_dict = {"error": "Internal server error", "errcode": Codes.UNKNOWN} 99 100 logger.error( 101 "Failed handle request via %r: %r", 102 request.request_metrics.name, 103 request, 104 exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] 105 ) 106 107 # Only respond with an error response if we haven't already started writing, 108 # otherwise lets just kill the connection 109 if request.startedWriting: 110 if request.transport: 111 try: 112 request.transport.abortConnection() 113 except Exception: 114 # abortConnection throws if the connection is already closed 115 pass 116 else: 117 respond_with_json( 118 request, 119 error_code, 120 error_dict, 121 send_cors=True, 122 ) 123 124 125def return_html_error( 126 f: failure.Failure, 127 request: Request, 128 error_template: Union[str, jinja2.Template], 129) -> None: 130 """Sends an HTML error page corresponding to the given failure. 131 132 Handles RedirectException and other CodeMessageExceptions (such as SynapseError) 133 134 Args: 135 f: the error to report 136 request: the failing request 137 error_template: the HTML template. Can be either a string (with `{code}`, 138 `{msg}` placeholders), or a jinja2 template 139 """ 140 if f.check(CodeMessageException): 141 # mypy doesn't understand that f.check asserts the type. 142 cme: CodeMessageException = f.value # type: ignore 143 code = cme.code 144 msg = cme.msg 145 146 if isinstance(cme, RedirectException): 147 logger.info("%s redirect to %s", request, cme.location) 148 request.setHeader(b"location", cme.location) 149 request.cookies.extend(cme.cookies) 150 elif isinstance(cme, SynapseError): 151 logger.info("%s SynapseError: %s - %s", request, code, msg) 152 else: 153 logger.error( 154 "Failed handle request %r", 155 request, 156 exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] 157 ) 158 else: 159 code = HTTPStatus.INTERNAL_SERVER_ERROR 160 msg = "Internal server error" 161 162 logger.error( 163 "Failed handle request %r", 164 request, 165 exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] 166 ) 167 168 if isinstance(error_template, str): 169 body = error_template.format(code=code, msg=html.escape(msg)) 170 else: 171 body = error_template.render(code=code, msg=msg) 172 173 respond_with_html(request, code, body) 174 175 176def wrap_async_request_handler( 177 h: Callable[["_AsyncResource", SynapseRequest], Awaitable[None]] 178) -> Callable[["_AsyncResource", SynapseRequest], "defer.Deferred[None]"]: 179 """Wraps an async request handler so that it calls request.processing. 180 181 This helps ensure that work done by the request handler after the request is completed 182 is correctly recorded against the request metrics/logs. 183 184 The handler method must have a signature of "handle_foo(self, request)", 185 where "request" must be a SynapseRequest. 186 187 The handler may return a deferred, in which case the completion of the request isn't 188 logged until the deferred completes. 189 """ 190 191 async def wrapped_async_request_handler( 192 self: "_AsyncResource", request: SynapseRequest 193 ) -> None: 194 with request.processing(): 195 await h(self, request) 196 197 # we need to preserve_fn here, because the synchronous render method won't yield for 198 # us (obviously) 199 return preserve_fn(wrapped_async_request_handler) 200 201 202# Type of a callback method for processing requests 203# it is actually called with a SynapseRequest and a kwargs dict for the params, 204# but I can't figure out how to represent that. 205ServletCallback = Callable[ 206 ..., Union[None, Awaitable[None], Tuple[int, Any], Awaitable[Tuple[int, Any]]] 207] 208 209 210class HttpServer(Protocol): 211 """Interface for registering callbacks on a HTTP server""" 212 213 def register_paths( 214 self, 215 method: str, 216 path_patterns: Iterable[Pattern], 217 callback: ServletCallback, 218 servlet_classname: str, 219 ) -> None: 220 """Register a callback that gets fired if we receive a http request 221 with the given method for a path that matches the given regex. 222 223 If the regex contains groups these gets passed to the callback via 224 an unpacked tuple. 225 226 Args: 227 method: The HTTP method to listen to. 228 path_patterns: The regex used to match requests. 229 callback: The function to fire if we receive a matched 230 request. The first argument will be the request object and 231 subsequent arguments will be any matched groups from the regex. 232 This should return either tuple of (code, response), or None. 233 servlet_classname (str): The name of the handler to be used in prometheus 234 and opentracing logs. 235 """ 236 pass 237 238 239class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): 240 """Base class for resources that have async handlers. 241 242 Sub classes can either implement `_async_render_<METHOD>` to handle 243 requests by method, or override `_async_render` to handle all requests. 244 245 Args: 246 extract_context: Whether to attempt to extract the opentracing 247 context from the request the servlet is handling. 248 """ 249 250 def __init__(self, extract_context: bool = False): 251 super().__init__() 252 253 self._extract_context = extract_context 254 255 def render(self, request: SynapseRequest) -> int: 256 """This gets called by twisted every time someone sends us a request.""" 257 defer.ensureDeferred(self._async_render_wrapper(request)) 258 return NOT_DONE_YET 259 260 @wrap_async_request_handler 261 async def _async_render_wrapper(self, request: SynapseRequest) -> None: 262 """This is a wrapper that delegates to `_async_render` and handles 263 exceptions, return values, metrics, etc. 264 """ 265 try: 266 request.request_metrics.name = self.__class__.__name__ 267 268 with trace_servlet(request, self._extract_context): 269 callback_return = await self._async_render(request) 270 271 if callback_return is not None: 272 code, response = callback_return 273 self._send_response(request, code, response) 274 except Exception: 275 # failure.Failure() fishes the original Failure out 276 # of our stack, and thus gives us a sensible stack 277 # trace. 278 f = failure.Failure() 279 self._send_error_response(f, request) 280 281 async def _async_render(self, request: SynapseRequest) -> Optional[Tuple[int, Any]]: 282 """Delegates to `_async_render_<METHOD>` methods, or returns a 400 if 283 no appropriate method exists. Can be overridden in sub classes for 284 different routing. 285 """ 286 # Treat HEAD requests as GET requests. 287 request_method = request.method.decode("ascii") 288 if request_method == "HEAD": 289 request_method = "GET" 290 291 method_handler = getattr(self, "_async_render_%s" % (request_method,), None) 292 if method_handler: 293 raw_callback_return = method_handler(request) 294 295 # Is it synchronous? We'll allow this for now. 296 if isawaitable(raw_callback_return): 297 callback_return = await raw_callback_return 298 else: 299 callback_return = raw_callback_return # type: ignore 300 301 return callback_return 302 303 _unrecognised_request_handler(request) 304 305 @abc.abstractmethod 306 def _send_response( 307 self, 308 request: SynapseRequest, 309 code: int, 310 response_object: Any, 311 ) -> None: 312 raise NotImplementedError() 313 314 @abc.abstractmethod 315 def _send_error_response( 316 self, 317 f: failure.Failure, 318 request: SynapseRequest, 319 ) -> None: 320 raise NotImplementedError() 321 322 323class DirectServeJsonResource(_AsyncResource): 324 """A resource that will call `self._async_on_<METHOD>` on new requests, 325 formatting responses and errors as JSON. 326 """ 327 328 def __init__(self, canonical_json: bool = False, extract_context: bool = False): 329 super().__init__(extract_context) 330 self.canonical_json = canonical_json 331 332 def _send_response( 333 self, 334 request: SynapseRequest, 335 code: int, 336 response_object: Any, 337 ) -> None: 338 """Implements _AsyncResource._send_response""" 339 # TODO: Only enable CORS for the requests that need it. 340 respond_with_json( 341 request, 342 code, 343 response_object, 344 send_cors=True, 345 canonical_json=self.canonical_json, 346 ) 347 348 def _send_error_response( 349 self, 350 f: failure.Failure, 351 request: SynapseRequest, 352 ) -> None: 353 """Implements _AsyncResource._send_error_response""" 354 return_json_error(f, request) 355 356 357@attr.s(slots=True, frozen=True, auto_attribs=True) 358class _PathEntry: 359 pattern: Pattern 360 callback: ServletCallback 361 servlet_classname: str 362 363 364class JsonResource(DirectServeJsonResource): 365 """This implements the HttpServer interface and provides JSON support for 366 Resources. 367 368 Register callbacks via register_paths() 369 370 Callbacks can return a tuple of status code and a dict in which case the 371 the dict will automatically be sent to the client as a JSON object. 372 373 The JsonResource is primarily intended for returning JSON, but callbacks 374 may send something other than JSON, they may do so by using the methods 375 on the request object and instead returning None. 376 """ 377 378 isLeaf = True 379 380 def __init__( 381 self, 382 hs: "HomeServer", 383 canonical_json: bool = True, 384 extract_context: bool = False, 385 ): 386 super().__init__(canonical_json, extract_context) 387 self.clock = hs.get_clock() 388 self.path_regexs: Dict[bytes, List[_PathEntry]] = {} 389 self.hs = hs 390 391 def register_paths( 392 self, 393 method: str, 394 path_patterns: Iterable[Pattern], 395 callback: ServletCallback, 396 servlet_classname: str, 397 ) -> None: 398 """ 399 Registers a request handler against a regular expression. Later request URLs are 400 checked against these regular expressions in order to identify an appropriate 401 handler for that request. 402 403 Args: 404 method: GET, POST etc 405 406 path_patterns: A list of regular expressions to which the request 407 URLs are compared. 408 409 callback: The handler for the request. Usually a Servlet 410 411 servlet_classname: The name of the handler to be used in prometheus 412 and opentracing logs. 413 """ 414 method_bytes = method.encode("utf-8") 415 416 for path_pattern in path_patterns: 417 logger.debug("Registering for %s %s", method, path_pattern.pattern) 418 self.path_regexs.setdefault(method_bytes, []).append( 419 _PathEntry(path_pattern, callback, servlet_classname) 420 ) 421 422 def _get_handler_for_request( 423 self, request: SynapseRequest 424 ) -> Tuple[ServletCallback, str, Dict[str, str]]: 425 """Finds a callback method to handle the given request. 426 427 Returns: 428 A tuple of the callback to use, the name of the servlet, and the 429 key word arguments to pass to the callback 430 """ 431 # At this point the path must be bytes. 432 request_path_bytes: bytes = request.path # type: ignore 433 request_path = request_path_bytes.decode("ascii") 434 # Treat HEAD requests as GET requests. 435 request_method = request.method 436 if request_method == b"HEAD": 437 request_method = b"GET" 438 439 # Loop through all the registered callbacks to check if the method 440 # and path regex match 441 for path_entry in self.path_regexs.get(request_method, []): 442 m = path_entry.pattern.match(request_path) 443 if m: 444 # We found a match! 445 return path_entry.callback, path_entry.servlet_classname, m.groupdict() 446 447 # Huh. No one wanted to handle that? Fiiiiiine. Send 400. 448 return _unrecognised_request_handler, "unrecognised_request_handler", {} 449 450 async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: 451 callback, servlet_classname, group_dict = self._get_handler_for_request(request) 452 453 # Make sure we have an appropriate name for this handler in prometheus 454 # (rather than the default of JsonResource). 455 request.request_metrics.name = servlet_classname 456 457 # Now trigger the callback. If it returns a response, we send it 458 # here. If it throws an exception, that is handled by the wrapper 459 # installed by @request_handler. 460 kwargs = intern_dict( 461 { 462 name: urllib.parse.unquote(value) if value else value 463 for name, value in group_dict.items() 464 } 465 ) 466 467 raw_callback_return = callback(request, **kwargs) 468 469 # Is it synchronous? We'll allow this for now. 470 if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)): 471 callback_return = await raw_callback_return 472 else: 473 callback_return = raw_callback_return # type: ignore 474 475 return callback_return 476 477 478class DirectServeHtmlResource(_AsyncResource): 479 """A resource that will call `self._async_on_<METHOD>` on new requests, 480 formatting responses and errors as HTML. 481 """ 482 483 # The error template to use for this resource 484 ERROR_TEMPLATE = HTML_ERROR_TEMPLATE 485 486 def _send_response( 487 self, 488 request: SynapseRequest, 489 code: int, 490 response_object: Any, 491 ) -> None: 492 """Implements _AsyncResource._send_response""" 493 # We expect to get bytes for us to write 494 assert isinstance(response_object, bytes) 495 html_bytes = response_object 496 497 respond_with_html_bytes(request, 200, html_bytes) 498 499 def _send_error_response( 500 self, 501 f: failure.Failure, 502 request: SynapseRequest, 503 ) -> None: 504 """Implements _AsyncResource._send_error_response""" 505 return_html_error(f, request, self.ERROR_TEMPLATE) 506 507 508class StaticResource(File): 509 """ 510 A resource that represents a plain non-interpreted file or directory. 511 512 Differs from the File resource by adding clickjacking protection. 513 """ 514 515 def render_GET(self, request: Request) -> bytes: 516 set_clickjacking_protection_headers(request) 517 return super().render_GET(request) 518 519 520def _unrecognised_request_handler(request: Request) -> NoReturn: 521 """Request handler for unrecognised requests 522 523 This is a request handler suitable for return from 524 _get_handler_for_request. It actually just raises an 525 UnrecognizedRequestError. 526 527 Args: 528 request: Unused, but passed in to match the signature of ServletCallback. 529 """ 530 raise UnrecognizedRequestError() 531 532 533class RootRedirect(resource.Resource): 534 """Redirects the root '/' path to another path.""" 535 536 def __init__(self, path: str): 537 super().__init__() 538 self.url = path 539 540 def render_GET(self, request: Request) -> bytes: 541 return redirectTo(self.url.encode("ascii"), request) 542 543 def getChild(self, name: str, request: Request) -> resource.Resource: 544 if len(name) == 0: 545 return self # select ourselves as the child to render 546 return super().getChild(name, request) 547 548 549class OptionsResource(resource.Resource): 550 """Responds to OPTION requests for itself and all children.""" 551 552 def render_OPTIONS(self, request: Request) -> bytes: 553 request.setResponseCode(204) 554 request.setHeader(b"Content-Length", b"0") 555 556 set_cors_headers(request) 557 558 return b"" 559 560 def getChildWithDefault(self, path: str, request: Request) -> resource.Resource: 561 if request.method == b"OPTIONS": 562 return self # select ourselves as the child to render 563 return super().getChildWithDefault(path, request) 564 565 566class RootOptionsRedirectResource(OptionsResource, RootRedirect): 567 pass 568 569 570@implementer(interfaces.IPushProducer) 571class _ByteProducer: 572 """ 573 Iteratively write bytes to the request. 574 """ 575 576 # The minimum number of bytes for each chunk. Note that the last chunk will 577 # usually be smaller than this. 578 min_chunk_size = 1024 579 580 def __init__( 581 self, 582 request: Request, 583 iterator: Iterator[bytes], 584 ): 585 self._request: Optional[Request] = request 586 self._iterator = iterator 587 self._paused = False 588 589 try: 590 self._request.registerProducer(self, True) 591 except AttributeError as e: 592 # Calling self._request.registerProducer might raise an AttributeError since 593 # the underlying Twisted code calls self._request.channel.registerProducer, 594 # however self._request.channel will be None if the connection was lost. 595 logger.info("Connection disconnected before response was written: %r", e) 596 597 # We drop our references to data we'll not use. 598 self._request = None 599 self._iterator = iter(()) 600 else: 601 # Start producing if `registerProducer` was successful 602 self.resumeProducing() 603 604 def _send_data(self, data: List[bytes]) -> None: 605 """ 606 Send a list of bytes as a chunk of a response. 607 """ 608 if not data or not self._request: 609 return 610 self._request.write(b"".join(data)) 611 612 def pauseProducing(self) -> None: 613 self._paused = True 614 615 def resumeProducing(self) -> None: 616 # We've stopped producing in the meantime (note that this might be 617 # re-entrant after calling write). 618 if not self._request: 619 return 620 621 self._paused = False 622 623 # Write until there's backpressure telling us to stop. 624 while not self._paused: 625 # Get the next chunk and write it to the request. 626 # 627 # The output of the JSON encoder is buffered and coalesced until 628 # min_chunk_size is reached. This is because JSON encoders produce 629 # very small output per iteration and the Request object converts 630 # each call to write() to a separate chunk. Without this there would 631 # be an explosion in bytes written (e.g. b"{" becoming "1\r\n{\r\n"). 632 # 633 # Note that buffer stores a list of bytes (instead of appending to 634 # bytes) to hopefully avoid many allocations. 635 buffer = [] 636 buffered_bytes = 0 637 while buffered_bytes < self.min_chunk_size: 638 try: 639 data = next(self._iterator) 640 buffer.append(data) 641 buffered_bytes += len(data) 642 except StopIteration: 643 # The entire JSON object has been serialized, write any 644 # remaining data, finalize the producer and the request, and 645 # clean-up any references. 646 self._send_data(buffer) 647 self._request.unregisterProducer() 648 self._request.finish() 649 self.stopProducing() 650 return 651 652 self._send_data(buffer) 653 654 def stopProducing(self) -> None: 655 # Clear a circular reference. 656 self._request = None 657 658 659def _encode_json_bytes(json_object: Any) -> bytes: 660 """ 661 Encode an object into JSON. Returns an iterator of bytes. 662 """ 663 return json_encoder.encode(json_object).encode("utf-8") 664 665 666def respond_with_json( 667 request: SynapseRequest, 668 code: int, 669 json_object: Any, 670 send_cors: bool = False, 671 canonical_json: bool = True, 672) -> Optional[int]: 673 """Sends encoded JSON in response to the given request. 674 675 Args: 676 request: The http request to respond to. 677 code: The HTTP response code. 678 json_object: The object to serialize to JSON. 679 send_cors: Whether to send Cross-Origin Resource Sharing headers 680 https://fetch.spec.whatwg.org/#http-cors-protocol 681 canonical_json: Whether to use the canonicaljson algorithm when encoding 682 the JSON bytes. 683 684 Returns: 685 twisted.web.server.NOT_DONE_YET if the request is still active. 686 """ 687 # could alternatively use request.notifyFinish() and flip a flag when 688 # the Deferred fires, but since the flag is RIGHT THERE it seems like 689 # a waste. 690 if request._disconnected: 691 logger.warning( 692 "Not sending response to request %s, already disconnected.", request 693 ) 694 return None 695 696 if canonical_json: 697 encoder = encode_canonical_json 698 else: 699 encoder = _encode_json_bytes 700 701 request.setResponseCode(code) 702 request.setHeader(b"Content-Type", b"application/json") 703 request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") 704 705 if send_cors: 706 set_cors_headers(request) 707 708 run_in_background( 709 _async_write_json_to_request_in_thread, request, encoder, json_object 710 ) 711 return NOT_DONE_YET 712 713 714def respond_with_json_bytes( 715 request: Request, 716 code: int, 717 json_bytes: bytes, 718 send_cors: bool = False, 719) -> Optional[int]: 720 """Sends encoded JSON in response to the given request. 721 722 Args: 723 request: The http request to respond to. 724 code: The HTTP response code. 725 json_bytes: The json bytes to use as the response body. 726 send_cors: Whether to send Cross-Origin Resource Sharing headers 727 https://fetch.spec.whatwg.org/#http-cors-protocol 728 729 Returns: 730 twisted.web.server.NOT_DONE_YET if the request is still active. 731 """ 732 if request._disconnected: 733 logger.warning( 734 "Not sending response to request %s, already disconnected.", request 735 ) 736 return None 737 738 request.setResponseCode(code) 739 request.setHeader(b"Content-Type", b"application/json") 740 request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) 741 request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") 742 743 if send_cors: 744 set_cors_headers(request) 745 746 _write_bytes_to_request(request, json_bytes) 747 return NOT_DONE_YET 748 749 750async def _async_write_json_to_request_in_thread( 751 request: SynapseRequest, 752 json_encoder: Callable[[Any], bytes], 753 json_object: Any, 754) -> None: 755 """Encodes the given JSON object on a thread and then writes it to the 756 request. 757 758 This is done so that encoding large JSON objects doesn't block the reactor 759 thread. 760 761 Note: We don't use JsonEncoder.iterencode here as that falls back to the 762 Python implementation (rather than the C backend), which is *much* more 763 expensive. 764 """ 765 766 def encode(opentracing_span: "Optional[opentracing.Span]") -> bytes: 767 # it might take a while for the threadpool to schedule us, so we write 768 # opentracing logs once we actually get scheduled, so that we can see how 769 # much that contributed. 770 if opentracing_span: 771 opentracing_span.log_kv({"event": "scheduled"}) 772 res = json_encoder(json_object) 773 if opentracing_span: 774 opentracing_span.log_kv({"event": "encoded"}) 775 return res 776 777 with start_active_span("encode_json_response"): 778 span = active_span() 779 json_str = await defer_to_thread(request.reactor, encode, span) 780 781 _write_bytes_to_request(request, json_str) 782 783 784def _write_bytes_to_request(request: Request, bytes_to_write: bytes) -> None: 785 """Writes the bytes to the request using an appropriate producer. 786 787 Note: This should be used instead of `Request.write` to correctly handle 788 large response bodies. 789 """ 790 791 # The problem with dumping all of the response into the `Request` object at 792 # once (via `Request.write`) is that doing so starts the timeout for the 793 # next request to be received: so if it takes longer than 60s to stream back 794 # the response to the client, the client never gets it. 795 # 796 # The correct solution is to use a Producer; then the timeout is only 797 # started once all of the content is sent over the TCP connection. 798 799 # To make sure we don't write all of the bytes at once we split it up into 800 # chunks. 801 chunk_size = 4096 802 bytes_generator = chunk_seq(bytes_to_write, chunk_size) 803 804 # We use a `_ByteProducer` here rather than `NoRangeStaticProducer` as the 805 # unit tests can't cope with being given a pull producer. 806 _ByteProducer(request, bytes_generator) 807 808 809def set_cors_headers(request: Request) -> None: 810 """Set the CORS headers so that javascript running in a web browsers can 811 use this API 812 813 Args: 814 request: The http request to add CORS to. 815 """ 816 request.setHeader(b"Access-Control-Allow-Origin", b"*") 817 request.setHeader( 818 b"Access-Control-Allow-Methods", b"GET, HEAD, POST, PUT, DELETE, OPTIONS" 819 ) 820 request.setHeader( 821 b"Access-Control-Allow-Headers", 822 b"X-Requested-With, Content-Type, Authorization, Date", 823 ) 824 825 826def respond_with_html(request: Request, code: int, html: str) -> None: 827 """ 828 Wraps `respond_with_html_bytes` by first encoding HTML from a str to UTF-8 bytes. 829 """ 830 respond_with_html_bytes(request, code, html.encode("utf-8")) 831 832 833def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> None: 834 """ 835 Sends HTML (encoded as UTF-8 bytes) as the response to the given request. 836 837 Note that this adds clickjacking protection headers and finishes the request. 838 839 Args: 840 request: The http request to respond to. 841 code: The HTTP response code. 842 html_bytes: The HTML bytes to use as the response body. 843 """ 844 # could alternatively use request.notifyFinish() and flip a flag when 845 # the Deferred fires, but since the flag is RIGHT THERE it seems like 846 # a waste. 847 if request._disconnected: 848 logger.warning( 849 "Not sending response to request %s, already disconnected.", request 850 ) 851 return None 852 853 request.setResponseCode(code) 854 request.setHeader(b"Content-Type", b"text/html; charset=utf-8") 855 request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) 856 857 # Ensure this content cannot be embedded. 858 set_clickjacking_protection_headers(request) 859 860 request.write(html_bytes) 861 finish_request(request) 862 863 864def set_clickjacking_protection_headers(request: Request) -> None: 865 """ 866 Set headers to guard against clickjacking of embedded content. 867 868 This sets the X-Frame-Options and Content-Security-Policy headers which instructs 869 browsers to not allow the HTML of the response to be embedded onto another 870 page. 871 872 Args: 873 request: The http request to add the headers to. 874 """ 875 request.setHeader(b"X-Frame-Options", b"DENY") 876 request.setHeader(b"Content-Security-Policy", b"frame-ancestors 'none';") 877 878 879def respond_with_redirect(request: Request, url: bytes) -> None: 880 """Write a 302 response to the request, if it is still alive.""" 881 logger.debug("Redirect to %s", url.decode("utf-8")) 882 request.redirect(url) 883 finish_request(request) 884 885 886def finish_request(request: Request) -> None: 887 """Finish writing the response to the request. 888 889 Twisted throws a RuntimeException if the connection closed before the 890 response was written but doesn't provide a convenient or reliable way to 891 determine if the connection was closed. So we catch and log the RuntimeException 892 893 You might think that ``request.notifyFinish`` could be used to tell if the 894 request was finished. However the deferred it returns won't fire if the 895 connection was already closed, meaning we'd have to have called the method 896 right at the start of the request. By the time we want to write the response 897 it will already be too late. 898 """ 899 try: 900 request.finish() 901 except RuntimeError as e: 902 logger.info("Connection disconnected before response was written: %r", e) 903