1# -*- test-case-name: twisted.test.test_amp -*- 2# Copyright (c) 2005 Divmod, Inc. 3# Copyright (c) Twisted Matrix Laboratories. 4# See LICENSE for details. 5 6""" 7This module implements AMP, the Asynchronous Messaging Protocol. 8 9AMP is a protocol for sending multiple asynchronous request/response pairs over 10the same connection. Requests and responses are both collections of key/value 11pairs. 12 13AMP is a very simple protocol which is not an application. This module is a 14"protocol construction kit" of sorts; it attempts to be the simplest wire-level 15implementation of Deferreds. AMP provides the following base-level features: 16 17 - Asynchronous request/response handling (hence the name) 18 19 - Requests and responses are both key/value pairs 20 21 - Binary transfer of all data: all data is length-prefixed. Your 22 application will never need to worry about quoting. 23 24 - Command dispatching (like HTTP Verbs): the protocol is extensible, and 25 multiple AMP sub-protocols can be grouped together easily. 26 27The protocol implementation also provides a few additional features which are 28not part of the core wire protocol, but are nevertheless very useful: 29 30 - Tight TLS integration, with an included StartTLS command. 31 32 - Handshaking to other protocols: because AMP has well-defined message 33 boundaries and maintains all incoming and outgoing requests for you, you 34 can start a connection over AMP and then switch to another protocol. 35 This makes it ideal for firewall-traversal applications where you may 36 have only one forwarded port but multiple applications that want to use 37 it. 38 39Using AMP with Twisted is simple. Each message is a command, with a response. 40You begin by defining a command type. Commands specify their input and output 41in terms of the types that they expect to see in the request and response 42key-value pairs. Here's an example of a command that adds two integers, 'a' 43and 'b':: 44 45 class Sum(amp.Command): 46 arguments = [('a', amp.Integer()), 47 ('b', amp.Integer())] 48 response = [('total', amp.Integer())] 49 50Once you have specified a command, you need to make it part of a protocol, and 51define a responder for it. Here's a 'JustSum' protocol that includes a 52responder for our 'Sum' command:: 53 54 class JustSum(amp.AMP): 55 def sum(self, a, b): 56 total = a + b 57 print 'Did a sum: %d + %d = %d' % (a, b, total) 58 return {'total': total} 59 Sum.responder(sum) 60 61Later, when you want to actually do a sum, the following expression will return 62a L{Deferred} which will fire with the result:: 63 64 ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback( 65 lambda p: p.callRemote(Sum, a=13, b=81)).addCallback( 66 lambda result: result['total']) 67 68Command responders may also return Deferreds, causing the response to be 69sent only once the Deferred fires:: 70 71 class DelayedSum(amp.AMP): 72 def slowSum(self, a, b): 73 total = a + b 74 result = defer.Deferred() 75 reactor.callLater(3, result.callback, {'total': total}) 76 return result 77 Sum.responder(slowSum) 78 79This is transparent to the caller. 80 81You can also define the propagation of specific errors in AMP. For example, 82for the slightly more complicated case of division, we might have to deal with 83division by zero:: 84 85 class Divide(amp.Command): 86 arguments = [('numerator', amp.Integer()), 87 ('denominator', amp.Integer())] 88 response = [('result', amp.Float())] 89 errors = {ZeroDivisionError: 'ZERO_DIVISION'} 90 91The 'errors' mapping here tells AMP that if a responder to Divide emits a 92L{ZeroDivisionError}, then the other side should be informed that an error of 93the type 'ZERO_DIVISION' has occurred. Writing a responder which takes 94advantage of this is very simple - just raise your exception normally:: 95 96 class JustDivide(amp.AMP): 97 def divide(self, numerator, denominator): 98 result = numerator / denominator 99 print 'Divided: %d / %d = %d' % (numerator, denominator, total) 100 return {'result': result} 101 Divide.responder(divide) 102 103On the client side, the errors mapping will be used to determine what the 104'ZERO_DIVISION' error means, and translated into an asynchronous exception, 105which can be handled normally as any L{Deferred} would be:: 106 107 def trapZero(result): 108 result.trap(ZeroDivisionError) 109 print "Divided by zero: returning INF" 110 return 1e1000 111 ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback( 112 lambda p: p.callRemote(Divide, numerator=1234, 113 denominator=0) 114 ).addErrback(trapZero) 115 116For a complete, runnable example of both of these commands, see the files in 117the Twisted repository:: 118 119 doc/core/examples/ampserver.py 120 doc/core/examples/ampclient.py 121 122On the wire, AMP is a protocol which uses 2-byte lengths to prefix keys and 123values, and empty keys to separate messages:: 124 125 <2-byte length><key><2-byte length><value> 126 <2-byte length><key><2-byte length><value> 127 ... 128 <2-byte length><key><2-byte length><value> 129 <NUL><NUL> # Empty Key == End of Message 130 131And so on. Because it's tedious to refer to lengths and NULs constantly, the 132documentation will refer to packets as if they were newline delimited, like 133so:: 134 135 C: _command: sum 136 C: _ask: ef639e5c892ccb54 137 C: a: 13 138 C: b: 81 139 140 S: _answer: ef639e5c892ccb54 141 S: total: 94 142 143Notes: 144 145In general, the order of keys is arbitrary. Specific uses of AMP may impose an 146ordering requirement, but unless this is specified explicitly, any ordering may 147be generated and any ordering must be accepted. This applies to the 148command-related keys I{_command} and I{_ask} as well as any other keys. 149 150Values are limited to the maximum encodable size in a 16-bit length, 65535 151bytes. 152 153Keys are limited to the maximum encodable size in a 8-bit length, 255 bytes. 154Note that we still use 2-byte lengths to encode keys. This small redundancy 155has several features: 156 157 - If an implementation becomes confused and starts emitting corrupt data, 158 or gets keys confused with values, many common errors will be signalled 159 immediately instead of delivering obviously corrupt packets. 160 161 - A single NUL will separate every key, and a double NUL separates 162 messages. This provides some redundancy when debugging traffic dumps. 163 164 - NULs will be present at regular intervals along the protocol, providing 165 some padding for otherwise braindead C implementations of the protocol, 166 so that <stdio.h> string functions will see the NUL and stop. 167 168 - This makes it possible to run an AMP server on a port also used by a 169 plain-text protocol, and easily distinguish between non-AMP clients (like 170 web browsers) which issue non-NUL as the first byte, and AMP clients, 171 which always issue NUL as the first byte. 172 173@var MAX_VALUE_LENGTH: The maximum length of a message. 174@type MAX_VALUE_LENGTH: L{int} 175 176@var ASK: Marker for an Ask packet. 177@type ASK: L{bytes} 178 179@var ANSWER: Marker for an Answer packet. 180@type ANSWER: L{bytes} 181 182@var COMMAND: Marker for a Command packet. 183@type COMMAND: L{bytes} 184 185@var ERROR: Marker for an AMP box of error type. 186@type ERROR: L{bytes} 187 188@var ERROR_CODE: Marker for an AMP box containing the code of an error. 189@type ERROR_CODE: L{bytes} 190 191@var ERROR_DESCRIPTION: Marker for an AMP box containing the description of the 192 error. 193@type ERROR_DESCRIPTION: L{bytes} 194""" 195 196 197import datetime 198import decimal 199import warnings 200from functools import partial 201from io import BytesIO 202from itertools import count 203from struct import pack 204from types import MethodType 205from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union 206 207from zope.interface import Interface, implementer 208 209from twisted.internet.defer import Deferred, fail, maybeDeferred 210from twisted.internet.error import ConnectionClosed, ConnectionLost, PeerVerifyError 211from twisted.internet.interfaces import IFileDescriptorReceiver 212from twisted.internet.main import CONNECTION_LOST 213from twisted.internet.protocol import Protocol 214from twisted.protocols.basic import Int16StringReceiver, StatefulStringProtocol 215from twisted.python import filepath, log 216from twisted.python._tzhelper import ( 217 UTC as utc, 218 FixedOffsetTimeZone as _FixedOffsetTZInfo, 219) 220from twisted.python.compat import nativeString 221from twisted.python.failure import Failure 222from twisted.python.reflect import accumulateClassDict 223 224try: 225 from twisted.internet import ssl as _ssl 226 227 if _ssl.supported: 228 from twisted.internet.ssl import DN, Certificate, CertificateOptions, KeyPair 229 else: 230 ssl = None 231except ImportError: 232 ssl = None 233else: 234 ssl = _ssl 235 236 237__all__ = [ 238 "AMP", 239 "ANSWER", 240 "ASK", 241 "AmpBox", 242 "AmpError", 243 "AmpList", 244 "Argument", 245 "BadLocalReturn", 246 "BinaryBoxProtocol", 247 "Boolean", 248 "Box", 249 "BoxDispatcher", 250 "COMMAND", 251 "Command", 252 "CommandLocator", 253 "Decimal", 254 "Descriptor", 255 "ERROR", 256 "ERROR_CODE", 257 "ERROR_DESCRIPTION", 258 "Float", 259 "IArgumentType", 260 "IBoxReceiver", 261 "IBoxSender", 262 "IResponderLocator", 263 "IncompatibleVersions", 264 "Integer", 265 "InvalidSignature", 266 "ListOf", 267 "MAX_KEY_LENGTH", 268 "MAX_VALUE_LENGTH", 269 "MalformedAmpBox", 270 "NoEmptyBoxes", 271 "OnlyOneTLS", 272 "PROTOCOL_ERRORS", 273 "PYTHON_KEYWORDS", 274 "Path", 275 "ProtocolSwitchCommand", 276 "ProtocolSwitched", 277 "QuitBox", 278 "RemoteAmpError", 279 "SimpleStringLocator", 280 "StartTLS", 281 "String", 282 "TooLong", 283 "UNHANDLED_ERROR_CODE", 284 "UNKNOWN_ERROR_CODE", 285 "UnhandledCommand", 286 "utc", 287 "Unicode", 288 "UnknownRemoteError", 289 "parse", 290 "parseString", 291] 292 293 294ASK = b"_ask" 295ANSWER = b"_answer" 296COMMAND = b"_command" 297ERROR = b"_error" 298ERROR_CODE = b"_error_code" 299ERROR_DESCRIPTION = b"_error_description" 300UNKNOWN_ERROR_CODE = b"UNKNOWN" 301UNHANDLED_ERROR_CODE = b"UNHANDLED" 302 303MAX_KEY_LENGTH = 0xFF 304MAX_VALUE_LENGTH = 0xFFFF 305 306 307class IArgumentType(Interface): 308 """ 309 An L{IArgumentType} can serialize a Python object into an AMP box and 310 deserialize information from an AMP box back into a Python object. 311 312 @since: 9.0 313 """ 314 315 def fromBox(name, strings, objects, proto): 316 """ 317 Given an argument name and an AMP box containing serialized values, 318 extract one or more Python objects and add them to the C{objects} 319 dictionary. 320 321 @param name: The name associated with this argument. Most commonly 322 this is the key which can be used to find a serialized value in 323 C{strings}. 324 @type name: C{bytes} 325 326 @param strings: The AMP box from which to extract one or more 327 values. 328 @type strings: C{dict} 329 330 @param objects: The output dictionary to populate with the value for 331 this argument. The key used will be derived from C{name}. It may 332 differ; in Python 3, for example, the key will be a Unicode/native 333 string. See L{_wireNameToPythonIdentifier}. 334 @type objects: C{dict} 335 336 @param proto: The protocol instance which received the AMP box being 337 interpreted. Most likely this is an instance of L{AMP}, but 338 this is not guaranteed. 339 340 @return: L{None} 341 """ 342 343 def toBox(name, strings, objects, proto): 344 """ 345 Given an argument name and a dictionary containing structured Python 346 objects, serialize values into one or more strings and add them to 347 the C{strings} dictionary. 348 349 @param name: The name associated with this argument. Most commonly 350 this is the key in C{strings} to associate with a C{bytes} giving 351 the serialized form of that object. 352 @type name: C{bytes} 353 354 @param strings: The AMP box into which to insert one or more strings. 355 @type strings: C{dict} 356 357 @param objects: The input dictionary from which to extract Python 358 objects to serialize. The key used will be derived from C{name}. 359 It may differ; in Python 3, for example, the key will be a 360 Unicode/native string. See L{_wireNameToPythonIdentifier}. 361 @type objects: C{dict} 362 363 @param proto: The protocol instance which will send the AMP box once 364 it is fully populated. Most likely this is an instance of 365 L{AMP}, but this is not guaranteed. 366 367 @return: L{None} 368 """ 369 370 371class IBoxSender(Interface): 372 """ 373 A transport which can send L{AmpBox} objects. 374 """ 375 376 def sendBox(box): 377 """ 378 Send an L{AmpBox}. 379 380 @raise ProtocolSwitched: if the underlying protocol has been 381 switched. 382 383 @raise ConnectionLost: if the underlying connection has already been 384 lost. 385 """ 386 387 def unhandledError(failure): 388 """ 389 An unhandled error occurred in response to a box. Log it 390 appropriately. 391 392 @param failure: a L{Failure} describing the error that occurred. 393 """ 394 395 396class IBoxReceiver(Interface): 397 """ 398 An application object which can receive L{AmpBox} objects and dispatch them 399 appropriately. 400 """ 401 402 def startReceivingBoxes(boxSender): 403 """ 404 The L{IBoxReceiver.ampBoxReceived} method will start being called; 405 boxes may be responded to by responding to the given L{IBoxSender}. 406 407 @param boxSender: an L{IBoxSender} provider. 408 """ 409 410 def ampBoxReceived(box): 411 """ 412 A box was received from the transport; dispatch it appropriately. 413 """ 414 415 def stopReceivingBoxes(reason): 416 """ 417 No further boxes will be received on this connection. 418 419 @type reason: L{Failure} 420 """ 421 422 423class IResponderLocator(Interface): 424 """ 425 An application object which can look up appropriate responder methods for 426 AMP commands. 427 """ 428 429 def locateResponder(name): 430 """ 431 Locate a responder method appropriate for the named command. 432 433 @param name: the wire-level name (commandName) of the AMP command to be 434 responded to. 435 @type name: C{bytes} 436 437 @return: a 1-argument callable that takes an L{AmpBox} with argument 438 values for the given command, and returns an L{AmpBox} containing 439 argument values for the named command, or a L{Deferred} that fires the 440 same. 441 """ 442 443 444class AmpError(Exception): 445 """ 446 Base class of all Amp-related exceptions. 447 """ 448 449 450class ProtocolSwitched(Exception): 451 """ 452 Connections which have been switched to other protocols can no longer 453 accept traffic at the AMP level. This is raised when you try to send it. 454 """ 455 456 457class OnlyOneTLS(AmpError): 458 """ 459 This is an implementation limitation; TLS may only be started once per 460 connection. 461 """ 462 463 464class NoEmptyBoxes(AmpError): 465 """ 466 You can't have empty boxes on the connection. This is raised when you 467 receive or attempt to send one. 468 """ 469 470 471class InvalidSignature(AmpError): 472 """ 473 You didn't pass all the required arguments. 474 """ 475 476 477class TooLong(AmpError): 478 """ 479 One of the protocol's length limitations was violated. 480 481 @ivar isKey: true if the string being encoded in a key position, false if 482 it was in a value position. 483 484 @ivar isLocal: Was the string encoded locally, or received too long from 485 the network? (It's only physically possible to encode "too long" values on 486 the network for keys.) 487 488 @ivar value: The string that was too long. 489 490 @ivar keyName: If the string being encoded was in a value position, what 491 key was it being encoded for? 492 """ 493 494 def __init__(self, isKey, isLocal, value, keyName=None): 495 AmpError.__init__(self) 496 self.isKey = isKey 497 self.isLocal = isLocal 498 self.value = value 499 self.keyName = keyName 500 501 def __repr__(self) -> str: 502 hdr = self.isKey and "key" or "value" 503 if not self.isKey: 504 hdr += " " + repr(self.keyName) 505 lcl = self.isLocal and "local" or "remote" 506 return "%s %s too long: %d" % (lcl, hdr, len(self.value)) 507 508 509class BadLocalReturn(AmpError): 510 """ 511 A bad value was returned from a local command; we were unable to coerce it. 512 """ 513 514 def __init__(self, message: str, enclosed: Failure) -> None: 515 AmpError.__init__(self) 516 self.message = message 517 self.enclosed = enclosed 518 519 def __repr__(self) -> str: 520 return self.message + " " + self.enclosed.getBriefTraceback() 521 522 __str__ = __repr__ 523 524 525class RemoteAmpError(AmpError): 526 """ 527 This error indicates that something went wrong on the remote end of the 528 connection, and the error was serialized and transmitted to you. 529 """ 530 531 def __init__(self, errorCode, description, fatal=False, local=None): 532 """Create a remote error with an error code and description. 533 534 @param errorCode: the AMP error code of this error. 535 @type errorCode: C{bytes} 536 537 @param description: some text to show to the user. 538 @type description: C{str} 539 540 @param fatal: a boolean, true if this error should terminate the 541 connection. 542 543 @param local: a local Failure, if one exists. 544 """ 545 if local: 546 localwhat = " (local)" 547 othertb = local.getBriefTraceback() 548 else: 549 localwhat = "" 550 othertb = "" 551 552 # Backslash-escape errorCode. Python 3.5 can do this natively 553 # ("backslashescape") but Python 2.7 and Python 3.4 can't. 554 errorCodeForMessage = "".join( 555 f"\\x{c:2x}" if c >= 0x80 else chr(c) for c in errorCode 556 ) 557 558 if othertb: 559 message = "Code<{}>{}: {}\n{}".format( 560 errorCodeForMessage, 561 localwhat, 562 description, 563 othertb, 564 ) 565 else: 566 message = "Code<{}>{}: {}".format( 567 errorCodeForMessage, localwhat, description 568 ) 569 570 super().__init__(message) 571 self.local = local 572 self.errorCode = errorCode 573 self.description = description 574 self.fatal = fatal 575 576 577class UnknownRemoteError(RemoteAmpError): 578 """ 579 This means that an error whose type we can't identify was raised from the 580 other side. 581 """ 582 583 def __init__(self, description): 584 errorCode = UNKNOWN_ERROR_CODE 585 RemoteAmpError.__init__(self, errorCode, description) 586 587 588class MalformedAmpBox(AmpError): 589 """ 590 This error indicates that the wire-level protocol was malformed. 591 """ 592 593 594class UnhandledCommand(AmpError): 595 """ 596 A command received via amp could not be dispatched. 597 """ 598 599 600class IncompatibleVersions(AmpError): 601 """ 602 It was impossible to negotiate a compatible version of the protocol with 603 the other end of the connection. 604 """ 605 606 607PROTOCOL_ERRORS = {UNHANDLED_ERROR_CODE: UnhandledCommand} 608 609 610class AmpBox(dict): 611 """ 612 I am a packet in the AMP protocol, much like a 613 regular bytes:bytes dictionary. 614 """ 615 616 # be like a regular dictionary don't magically 617 # acquire a __dict__... 618 __slots__: List[str] = [] 619 620 def __init__(self, *args, **kw): 621 """ 622 Initialize a new L{AmpBox}. 623 624 In Python 3, keyword arguments MUST be Unicode/native strings whereas 625 in Python 2 they could be either byte strings or Unicode strings. 626 627 However, all keys of an L{AmpBox} MUST be byte strings, or possible to 628 transparently coerce into byte strings (i.e. Python 2). 629 630 In Python 3, therefore, native string keys are coerced to byte strings 631 by encoding as ASCII. This can result in C{UnicodeEncodeError} being 632 raised. 633 634 @param args: See C{dict}, but all keys and values should be C{bytes}. 635 On Python 3, native strings may be used as keys provided they 636 contain only ASCII characters. 637 638 @param kw: See C{dict}, but all keys and values should be C{bytes}. 639 On Python 3, native strings may be used as keys provided they 640 contain only ASCII characters. 641 642 @raise UnicodeEncodeError: When a native string key cannot be coerced 643 to an ASCII byte string (Python 3 only). 644 """ 645 super().__init__(*args, **kw) 646 nonByteNames = [n for n in self if not isinstance(n, bytes)] 647 for nonByteName in nonByteNames: 648 byteName = nonByteName.encode("ascii") 649 self[byteName] = self.pop(nonByteName) 650 651 def copy(self): 652 """ 653 Return another AmpBox just like me. 654 """ 655 newBox = self.__class__() 656 newBox.update(self) 657 return newBox 658 659 def serialize(self): 660 """ 661 Convert me into a wire-encoded string. 662 663 @return: a C{bytes} encoded according to the rules described in the 664 module docstring. 665 """ 666 i = sorted(self.items()) 667 L = [] 668 w = L.append 669 for k, v in i: 670 if type(k) == str: 671 raise TypeError("Unicode key not allowed: %r" % k) 672 if type(v) == str: 673 raise TypeError(f"Unicode value for key {k!r} not allowed: {v!r}") 674 if len(k) > MAX_KEY_LENGTH: 675 raise TooLong(True, True, k, None) 676 if len(v) > MAX_VALUE_LENGTH: 677 raise TooLong(False, True, v, k) 678 for kv in k, v: 679 w(pack("!H", len(kv))) 680 w(kv) 681 w(pack("!H", 0)) 682 return b"".join(L) 683 684 def _sendTo(self, proto): 685 """ 686 Serialize and send this box to an Amp instance. By the time it is being 687 sent, several keys are required. I must have exactly ONE of:: 688 689 _ask 690 _answer 691 _error 692 693 If the '_ask' key is set, then the '_command' key must also be 694 set. 695 696 @param proto: an AMP instance. 697 """ 698 proto.sendBox(self) 699 700 def __repr__(self) -> str: 701 return f"AmpBox({dict.__repr__(self)})" 702 703 704# amp.Box => AmpBox 705 706Box = AmpBox 707 708 709class QuitBox(AmpBox): 710 """ 711 I am an AmpBox that, upon being sent, terminates the connection. 712 """ 713 714 __slots__: List[str] = [] 715 716 def __repr__(self) -> str: 717 return f"QuitBox(**{super().__repr__()})" 718 719 def _sendTo(self, proto): 720 """ 721 Immediately call loseConnection after sending. 722 """ 723 super()._sendTo(proto) 724 proto.transport.loseConnection() 725 726 727class _SwitchBox(AmpBox): 728 """ 729 Implementation detail of ProtocolSwitchCommand: I am an AmpBox which sets 730 up state for the protocol to switch. 731 """ 732 733 # DON'T set __slots__ here; we do have an attribute. 734 735 def __init__(self, innerProto, **kw): 736 """ 737 Create a _SwitchBox with the protocol to switch to after being sent. 738 739 @param innerProto: the protocol instance to switch to. 740 @type innerProto: an IProtocol provider. 741 """ 742 super().__init__(**kw) 743 self.innerProto = innerProto 744 745 def __repr__(self) -> str: 746 return "_SwitchBox({!r}, **{})".format( 747 self.innerProto, 748 dict.__repr__(self), 749 ) 750 751 def _sendTo(self, proto): 752 """ 753 Send me; I am the last box on the connection. All further traffic will be 754 over the new protocol. 755 """ 756 super()._sendTo(proto) 757 proto._lockForSwitch() 758 proto._switchTo(self.innerProto) 759 760 761@implementer(IBoxReceiver) 762class BoxDispatcher: 763 """ 764 A L{BoxDispatcher} dispatches '_ask', '_answer', and '_error' L{AmpBox}es, 765 both incoming and outgoing, to their appropriate destinations. 766 767 Outgoing commands are converted into L{Deferred}s and outgoing boxes, and 768 associated tracking state to fire those L{Deferred} when '_answer' boxes 769 come back. Incoming '_answer' and '_error' boxes are converted into 770 callbacks and errbacks on those L{Deferred}s, respectively. 771 772 Incoming '_ask' boxes are converted into method calls on a supplied method 773 locator. 774 775 @ivar _outstandingRequests: a dictionary mapping request IDs to 776 L{Deferred}s which were returned for those requests. 777 778 @ivar locator: an object with a L{CommandLocator.locateResponder} method 779 that locates a responder function that takes a Box and returns a result 780 (either a Box or a Deferred which fires one). 781 782 @ivar boxSender: an object which can send boxes, via the L{_sendBoxCommand} 783 method, such as an L{AMP} instance. 784 @type boxSender: L{IBoxSender} 785 """ 786 787 _failAllReason = None 788 _outstandingRequests = None 789 _counter = 0 790 boxSender = None 791 792 def __init__(self, locator): 793 self._outstandingRequests = {} 794 self.locator = locator 795 796 def startReceivingBoxes(self, boxSender): 797 """ 798 The given boxSender is going to start calling boxReceived on this 799 L{BoxDispatcher}. 800 801 @param boxSender: The L{IBoxSender} to send command responses to. 802 """ 803 self.boxSender = boxSender 804 805 def stopReceivingBoxes(self, reason): 806 """ 807 No further boxes will be received here. Terminate all currently 808 outstanding command deferreds with the given reason. 809 """ 810 self.failAllOutgoing(reason) 811 812 def failAllOutgoing(self, reason): 813 """ 814 Call the errback on all outstanding requests awaiting responses. 815 816 @param reason: the Failure instance to pass to those errbacks. 817 """ 818 self._failAllReason = reason 819 OR = self._outstandingRequests.items() 820 self._outstandingRequests = None # we can never send another request 821 for key, value in OR: 822 value.errback(reason) 823 824 def _nextTag(self): 825 """ 826 Generate protocol-local serial numbers for _ask keys. 827 828 @return: a string that has not yet been used on this connection. 829 """ 830 self._counter += 1 831 return b"%x" % (self._counter,) 832 833 def _sendBoxCommand(self, command, box, requiresAnswer=True): 834 """ 835 Send a command across the wire with the given C{amp.Box}. 836 837 Mutate the given box to give it any additional keys (_command, _ask) 838 required for the command and request/response machinery, then send it. 839 840 If requiresAnswer is True, returns a C{Deferred} which fires when a 841 response is received. The C{Deferred} is fired with an C{amp.Box} on 842 success, or with an C{amp.RemoteAmpError} if an error is received. 843 844 If the Deferred fails and the error is not handled by the caller of 845 this method, the failure will be logged and the connection dropped. 846 847 @param command: a C{bytes}, the name of the command to issue. 848 849 @param box: an AmpBox with the arguments for the command. 850 851 @param requiresAnswer: a boolean. Defaults to True. If True, return a 852 Deferred which will fire when the other side responds to this command. 853 If False, return None and do not ask the other side for acknowledgement. 854 855 @return: a Deferred which fires the AmpBox that holds the response to 856 this command, or None, as specified by requiresAnswer. 857 858 @raise ProtocolSwitched: if the protocol has been switched. 859 """ 860 if self._failAllReason is not None: 861 if requiresAnswer: 862 return fail(self._failAllReason) 863 else: 864 return None 865 box[COMMAND] = command 866 tag = self._nextTag() 867 if requiresAnswer: 868 box[ASK] = tag 869 box._sendTo(self.boxSender) 870 if requiresAnswer: 871 result = self._outstandingRequests[tag] = Deferred() 872 else: 873 result = None 874 return result 875 876 def callRemoteString(self, command, requiresAnswer=True, **kw): 877 """ 878 This is a low-level API, designed only for optimizing simple messages 879 for which the overhead of parsing is too great. 880 881 @param command: a C{bytes} naming the command. 882 883 @param kw: arguments to the amp box. 884 885 @param requiresAnswer: a boolean. Defaults to True. If True, return a 886 Deferred which will fire when the other side responds to this command. 887 If False, return None and do not ask the other side for acknowledgement. 888 889 @return: a Deferred which fires the AmpBox that holds the response to 890 this command, or None, as specified by requiresAnswer. 891 """ 892 box = Box(kw) 893 return self._sendBoxCommand(command, box, requiresAnswer) 894 895 def callRemote(self, commandType, *a, **kw): 896 """ 897 This is the primary high-level API for sending messages via AMP. Invoke it 898 with a command and appropriate arguments to send a message to this 899 connection's peer. 900 901 @param commandType: a subclass of Command. 902 @type commandType: L{type} 903 904 @param a: Positional (special) parameters taken by the command. 905 Positional parameters will typically not be sent over the wire. The 906 only command included with AMP which uses positional parameters is 907 L{ProtocolSwitchCommand}, which takes the protocol that will be 908 switched to as its first argument. 909 910 @param kw: Keyword arguments taken by the command. These are the 911 arguments declared in the command's 'arguments' attribute. They will 912 be encoded and sent to the peer as arguments for the L{commandType}. 913 914 @return: If L{commandType} has a C{requiresAnswer} attribute set to 915 L{False}, then return L{None}. Otherwise, return a L{Deferred} which 916 fires with a dictionary of objects representing the result of this 917 call. Additionally, this L{Deferred} may fail with an exception 918 representing a connection failure, with L{UnknownRemoteError} if the 919 other end of the connection fails for an unknown reason, or with any 920 error specified as a key in L{commandType}'s C{errors} dictionary. 921 """ 922 923 # XXX this takes command subclasses and not command objects on purpose. 924 # There's really no reason to have all this back-and-forth between 925 # command objects and the protocol, and the extra object being created 926 # (the Command instance) is pointless. Command is kind of like 927 # Interface, and should be more like it. 928 929 # In other words, the fact that commandType is instantiated here is an 930 # implementation detail. Don't rely on it. 931 932 try: 933 co = commandType(*a, **kw) 934 except BaseException: 935 return fail() 936 return co._doCommand(self) 937 938 def unhandledError(self, failure): 939 """ 940 This is a terminal callback called after application code has had a 941 chance to quash any errors. 942 """ 943 return self.boxSender.unhandledError(failure) 944 945 def _answerReceived(self, box): 946 """ 947 An AMP box was received that answered a command previously sent with 948 L{callRemote}. 949 950 @param box: an AmpBox with a value for its L{ANSWER} key. 951 """ 952 question = self._outstandingRequests.pop(box[ANSWER]) 953 question.addErrback(self.unhandledError) 954 question.callback(box) 955 956 def _errorReceived(self, box): 957 """ 958 An AMP box was received that answered a command previously sent with 959 L{callRemote}, with an error. 960 961 @param box: an L{AmpBox} with a value for its L{ERROR}, L{ERROR_CODE}, 962 and L{ERROR_DESCRIPTION} keys. 963 """ 964 question = self._outstandingRequests.pop(box[ERROR]) 965 question.addErrback(self.unhandledError) 966 errorCode = box[ERROR_CODE] 967 description = box[ERROR_DESCRIPTION] 968 if isinstance(description, bytes): 969 description = description.decode("utf-8", "replace") 970 if errorCode in PROTOCOL_ERRORS: 971 exc = PROTOCOL_ERRORS[errorCode](errorCode, description) 972 else: 973 exc = RemoteAmpError(errorCode, description) 974 question.errback(Failure(exc)) 975 976 def _commandReceived(self, box): 977 """ 978 @param box: an L{AmpBox} with a value for its L{COMMAND} and L{ASK} 979 keys. 980 """ 981 982 def formatAnswer(answerBox): 983 answerBox[ANSWER] = box[ASK] 984 return answerBox 985 986 def formatError(error): 987 if error.check(RemoteAmpError): 988 code = error.value.errorCode 989 desc = error.value.description 990 if isinstance(desc, str): 991 desc = desc.encode("utf-8", "replace") 992 if error.value.fatal: 993 errorBox = QuitBox() 994 else: 995 errorBox = AmpBox() 996 else: 997 errorBox = QuitBox() 998 log.err(error) # here is where server-side logging happens 999 # if the error isn't handled 1000 code = UNKNOWN_ERROR_CODE 1001 desc = b"Unknown Error" 1002 errorBox[ERROR] = box[ASK] 1003 errorBox[ERROR_DESCRIPTION] = desc 1004 errorBox[ERROR_CODE] = code 1005 return errorBox 1006 1007 deferred = self.dispatchCommand(box) 1008 if ASK in box: 1009 deferred.addCallbacks(formatAnswer, formatError) 1010 deferred.addCallback(self._safeEmit) 1011 deferred.addErrback(self.unhandledError) 1012 1013 def ampBoxReceived(self, box): 1014 """ 1015 An AmpBox was received, representing a command, or an answer to a 1016 previously issued command (either successful or erroneous). Respond to 1017 it according to its contents. 1018 1019 @param box: an AmpBox 1020 1021 @raise NoEmptyBoxes: when a box is received that does not contain an 1022 '_answer', '_command' / '_ask', or '_error' key; i.e. one which does not 1023 fit into the command / response protocol defined by AMP. 1024 """ 1025 if ANSWER in box: 1026 self._answerReceived(box) 1027 elif ERROR in box: 1028 self._errorReceived(box) 1029 elif COMMAND in box: 1030 self._commandReceived(box) 1031 else: 1032 raise NoEmptyBoxes(box) 1033 1034 def _safeEmit(self, aBox): 1035 """ 1036 Emit a box, ignoring L{ProtocolSwitched} and L{ConnectionLost} errors 1037 which cannot be usefully handled. 1038 """ 1039 try: 1040 aBox._sendTo(self.boxSender) 1041 except (ProtocolSwitched, ConnectionLost): 1042 pass 1043 1044 def dispatchCommand(self, box): 1045 """ 1046 A box with a _command key was received. 1047 1048 Dispatch it to a local handler call it. 1049 1050 @param box: an AmpBox to be dispatched. 1051 """ 1052 cmd = box[COMMAND] 1053 responder = self.locator.locateResponder(cmd) 1054 if responder is None: 1055 description = f"Unhandled Command: {cmd!r}" 1056 return fail( 1057 RemoteAmpError( 1058 UNHANDLED_ERROR_CODE, 1059 description, 1060 False, 1061 local=Failure(UnhandledCommand()), 1062 ) 1063 ) 1064 return maybeDeferred(responder, box) 1065 1066 1067class _CommandLocatorMeta(type): 1068 """ 1069 This metaclass keeps track of all of the Command.responder-decorated 1070 methods defined since the last CommandLocator subclass was defined. It 1071 assumes (usually correctly, but unfortunately not necessarily so) that 1072 those commands responders were all declared as methods of the class 1073 being defined. Note that this list can be incorrect if users use the 1074 Command.responder decorator outside the context of a CommandLocator 1075 class declaration. 1076 1077 Command responders defined on subclasses are given precedence over 1078 those inherited from a base class. 1079 1080 The Command.responder decorator explicitly cooperates with this 1081 metaclass. 1082 """ 1083 1084 _currentClassCommands: "List[Tuple[Command, Callable]]" = [] 1085 1086 def __new__(cls, name, bases, attrs): 1087 commands = cls._currentClassCommands[:] 1088 cls._currentClassCommands[:] = [] 1089 cd = attrs["_commandDispatch"] = {} 1090 subcls = type.__new__(cls, name, bases, attrs) 1091 ancestors = list(subcls.__mro__[1:]) 1092 ancestors.reverse() 1093 for ancestor in ancestors: 1094 cd.update(getattr(ancestor, "_commandDispatch", {})) 1095 for commandClass, responderFunc in commands: 1096 cd[commandClass.commandName] = (commandClass, responderFunc) 1097 if bases and (subcls.lookupFunction != CommandLocator.lookupFunction): 1098 1099 def locateResponder(self, name): 1100 warnings.warn( 1101 "Override locateResponder, not lookupFunction.", 1102 category=PendingDeprecationWarning, 1103 stacklevel=2, 1104 ) 1105 return self.lookupFunction(name) 1106 1107 subcls.locateResponder = locateResponder 1108 return subcls 1109 1110 1111@implementer(IResponderLocator) 1112class CommandLocator(metaclass=_CommandLocatorMeta): 1113 """ 1114 A L{CommandLocator} is a collection of responders to AMP L{Command}s, with 1115 the help of the L{Command.responder} decorator. 1116 """ 1117 1118 def _wrapWithSerialization(self, aCallable, command): 1119 """ 1120 Wrap aCallable with its command's argument de-serialization 1121 and result serialization logic. 1122 1123 @param aCallable: a callable with a 'command' attribute, designed to be 1124 called with keyword arguments. 1125 1126 @param command: the command class whose serialization to use. 1127 1128 @return: a 1-arg callable which, when invoked with an AmpBox, will 1129 deserialize the argument list and invoke appropriate user code for the 1130 callable's command, returning a Deferred which fires with the result or 1131 fails with an error. 1132 """ 1133 1134 def doit(box): 1135 kw = command.parseArguments(box, self) 1136 1137 def checkKnownErrors(error): 1138 key = error.trap(*command.allErrors) 1139 code = command.allErrors[key] 1140 desc = str(error.value) 1141 return Failure( 1142 RemoteAmpError(code, desc, key in command.fatalErrors, local=error) 1143 ) 1144 1145 def makeResponseFor(objects): 1146 try: 1147 return command.makeResponse(objects, self) 1148 except BaseException: 1149 # let's helpfully log this. 1150 originalFailure = Failure() 1151 raise BadLocalReturn( 1152 "%r returned %r and %r could not serialize it" 1153 % (aCallable, objects, command), 1154 originalFailure, 1155 ) 1156 1157 return ( 1158 maybeDeferred(aCallable, **kw) 1159 .addCallback(makeResponseFor) 1160 .addErrback(checkKnownErrors) 1161 ) 1162 1163 return doit 1164 1165 def lookupFunction(self, name): 1166 """ 1167 Deprecated synonym for L{CommandLocator.locateResponder} 1168 """ 1169 if self.__class__.lookupFunction != CommandLocator.lookupFunction: 1170 return CommandLocator.locateResponder(self, name) 1171 else: 1172 warnings.warn( 1173 "Call locateResponder, not lookupFunction.", 1174 category=PendingDeprecationWarning, 1175 stacklevel=2, 1176 ) 1177 return self.locateResponder(name) 1178 1179 def locateResponder(self, name): 1180 """ 1181 Locate a callable to invoke when executing the named command. 1182 1183 @param name: the normalized name (from the wire) of the command. 1184 @type name: C{bytes} 1185 1186 @return: a 1-argument function that takes a Box and returns a box or a 1187 Deferred which fires a Box, for handling the command identified by the 1188 given name, or None, if no appropriate responder can be found. 1189 """ 1190 # Try to find a high-level method to invoke, and if we can't find one, 1191 # fall back to a low-level one. 1192 cd = self._commandDispatch 1193 if name in cd: 1194 commandClass, responderFunc = cd[name] 1195 responderMethod = MethodType(responderFunc, self) 1196 return self._wrapWithSerialization(responderMethod, commandClass) 1197 1198 1199@implementer(IResponderLocator) 1200class SimpleStringLocator: 1201 """ 1202 Implement the L{AMP.locateResponder} method to do simple, string-based 1203 dispatch. 1204 """ 1205 1206 baseDispatchPrefix = b"amp_" 1207 1208 def locateResponder(self, name): 1209 """ 1210 Locate a callable to invoke when executing the named command. 1211 1212 @return: a function with the name C{"amp_" + name} on the same 1213 instance, or None if no such function exists. 1214 This function will then be called with the L{AmpBox} itself as an 1215 argument. 1216 1217 @param name: the normalized name (from the wire) of the command. 1218 @type name: C{bytes} 1219 """ 1220 fName = nativeString(self.baseDispatchPrefix + name.upper()) 1221 return getattr(self, fName, None) 1222 1223 1224PYTHON_KEYWORDS = [ 1225 "and", 1226 "del", 1227 "for", 1228 "is", 1229 "raise", 1230 "assert", 1231 "elif", 1232 "from", 1233 "lambda", 1234 "return", 1235 "break", 1236 "else", 1237 "global", 1238 "not", 1239 "try", 1240 "class", 1241 "except", 1242 "if", 1243 "or", 1244 "while", 1245 "continue", 1246 "exec", 1247 "import", 1248 "pass", 1249 "yield", 1250 "def", 1251 "finally", 1252 "in", 1253 "print", 1254] 1255 1256 1257def _wireNameToPythonIdentifier(key): 1258 """ 1259 (Private) Normalize an argument name from the wire for use with Python 1260 code. If the return value is going to be a python keyword it will be 1261 capitalized. If it contains any dashes they will be replaced with 1262 underscores. 1263 1264 The rationale behind this method is that AMP should be an inherently 1265 multi-language protocol, so message keys may contain all manner of bizarre 1266 bytes. This is not a complete solution; there are still forms of arguments 1267 that this implementation will be unable to parse. However, Python 1268 identifiers share a huge raft of properties with identifiers from many 1269 other languages, so this is a 'good enough' effort for now. We deal 1270 explicitly with dashes because that is the most likely departure: Lisps 1271 commonly use dashes to separate method names, so protocols initially 1272 implemented in a lisp amp dialect may use dashes in argument or command 1273 names. 1274 1275 @param key: a C{bytes}, looking something like 'foo-bar-baz' or 'from' 1276 @type key: C{bytes} 1277 1278 @return: a native string which is a valid python identifier, looking 1279 something like 'foo_bar_baz' or 'From'. 1280 """ 1281 lkey = nativeString(key.replace(b"-", b"_")) 1282 if lkey in PYTHON_KEYWORDS: 1283 return lkey.title() 1284 return lkey 1285 1286 1287@implementer(IArgumentType) 1288class Argument: 1289 """ 1290 Base-class of all objects that take values from Amp packets and convert 1291 them into objects for Python functions. 1292 1293 This implementation of L{IArgumentType} provides several higher-level 1294 hooks for subclasses to override. See L{toString} and L{fromString} 1295 which will be used to define the behavior of L{IArgumentType.toBox} and 1296 L{IArgumentType.fromBox}, respectively. 1297 """ 1298 1299 optional = False 1300 1301 def __init__(self, optional=False): 1302 """ 1303 Create an Argument. 1304 1305 @param optional: a boolean indicating whether this argument can be 1306 omitted in the protocol. 1307 """ 1308 self.optional = optional 1309 1310 def retrieve(self, d, name, proto): 1311 """ 1312 Retrieve the given key from the given dictionary, removing it if found. 1313 1314 @param d: a dictionary. 1315 1316 @param name: a key in I{d}. 1317 1318 @param proto: an instance of an AMP. 1319 1320 @raise KeyError: if I am not optional and no value was found. 1321 1322 @return: d[name]. 1323 """ 1324 if self.optional: 1325 value = d.get(name) 1326 if value is not None: 1327 del d[name] 1328 else: 1329 value = d.pop(name) 1330 return value 1331 1332 def fromBox(self, name, strings, objects, proto): 1333 """ 1334 Populate an 'out' dictionary with mapping names to Python values 1335 decoded from an 'in' AmpBox mapping strings to string values. 1336 1337 @param name: the argument name to retrieve 1338 @type name: C{bytes} 1339 1340 @param strings: The AmpBox to read string(s) from, a mapping of 1341 argument names to string values. 1342 @type strings: AmpBox 1343 1344 @param objects: The dictionary to write object(s) to, a mapping of 1345 names to Python objects. Keys will be native strings. 1346 @type objects: dict 1347 1348 @param proto: an AMP instance. 1349 """ 1350 st = self.retrieve(strings, name, proto) 1351 nk = _wireNameToPythonIdentifier(name) 1352 if self.optional and st is None: 1353 objects[nk] = None 1354 else: 1355 objects[nk] = self.fromStringProto(st, proto) 1356 1357 def toBox(self, name, strings, objects, proto): 1358 """ 1359 Populate an 'out' AmpBox with strings encoded from an 'in' dictionary 1360 mapping names to Python values. 1361 1362 @param name: the argument name to retrieve 1363 @type name: C{bytes} 1364 1365 @param strings: The AmpBox to write string(s) to, a mapping of 1366 argument names to string values. 1367 @type strings: AmpBox 1368 1369 @param objects: The dictionary to read object(s) from, a mapping of 1370 names to Python objects. Keys should be native strings. 1371 1372 @type objects: dict 1373 1374 @param proto: the protocol we are converting for. 1375 @type proto: AMP 1376 """ 1377 obj = self.retrieve(objects, _wireNameToPythonIdentifier(name), proto) 1378 if self.optional and obj is None: 1379 # strings[name] = None 1380 pass 1381 else: 1382 strings[name] = self.toStringProto(obj, proto) 1383 1384 def fromStringProto(self, inString, proto): 1385 """ 1386 Convert a string to a Python value. 1387 1388 @param inString: the string to convert. 1389 @type inString: C{bytes} 1390 1391 @param proto: the protocol we are converting for. 1392 @type proto: AMP 1393 1394 @return: a Python object. 1395 """ 1396 return self.fromString(inString) 1397 1398 def toStringProto(self, inObject, proto): 1399 """ 1400 Convert a Python object to a string. 1401 1402 @param inObject: the object to convert. 1403 1404 @param proto: the protocol we are converting for. 1405 @type proto: AMP 1406 """ 1407 return self.toString(inObject) 1408 1409 def fromString(self, inString): 1410 """ 1411 Convert a string to a Python object. Subclasses must implement this. 1412 1413 @param inString: the string to convert. 1414 @type inString: C{bytes} 1415 1416 @return: the decoded value from C{inString} 1417 """ 1418 1419 def toString(self, inObject): 1420 """ 1421 Convert a Python object into a string for passing over the network. 1422 1423 @param inObject: an object of the type that this Argument is intended 1424 to deal with. 1425 1426 @return: the wire encoding of inObject 1427 @rtype: C{bytes} 1428 """ 1429 1430 1431class Integer(Argument): 1432 """ 1433 Encode any integer values of any size on the wire as the string 1434 representation. 1435 1436 Example: C{123} becomes C{"123"} 1437 """ 1438 1439 fromString = int 1440 1441 def toString(self, inObject): 1442 return b"%d" % (inObject,) 1443 1444 1445class String(Argument): 1446 """ 1447 Don't do any conversion at all; just pass through 'str'. 1448 """ 1449 1450 def toString(self, inObject): 1451 return inObject 1452 1453 def fromString(self, inString): 1454 return inString 1455 1456 1457class Float(Argument): 1458 """ 1459 Encode floating-point values on the wire as their repr. 1460 """ 1461 1462 fromString = float 1463 1464 def toString(self, inString): 1465 if not isinstance(inString, float): 1466 raise ValueError(f"Bad float value {inString!r}") 1467 return str(inString).encode("ascii") 1468 1469 1470class Boolean(Argument): 1471 """ 1472 Encode True or False as "True" or "False" on the wire. 1473 """ 1474 1475 def fromString(self, inString): 1476 if inString == b"True": 1477 return True 1478 elif inString == b"False": 1479 return False 1480 else: 1481 raise TypeError(f"Bad boolean value: {inString!r}") 1482 1483 def toString(self, inObject): 1484 if inObject: 1485 return b"True" 1486 else: 1487 return b"False" 1488 1489 1490class Unicode(String): 1491 """ 1492 Encode a unicode string on the wire as UTF-8. 1493 """ 1494 1495 def toString(self, inObject): 1496 return String.toString(self, inObject.encode("utf-8")) 1497 1498 def fromString(self, inString): 1499 return String.fromString(self, inString).decode("utf-8") 1500 1501 1502class Path(Unicode): 1503 """ 1504 Encode and decode L{filepath.FilePath} instances as paths on the wire. 1505 1506 This is really intended for use with subprocess communication tools: 1507 exchanging pathnames on different machines over a network is not generally 1508 meaningful, but neither is it disallowed; you can use this to communicate 1509 about NFS paths, for example. 1510 """ 1511 1512 def fromString(self, inString): 1513 return filepath.FilePath(Unicode.fromString(self, inString)) 1514 1515 def toString(self, inObject): 1516 return Unicode.toString(self, inObject.asTextMode().path) 1517 1518 1519class ListOf(Argument): 1520 """ 1521 Encode and decode lists of instances of a single other argument type. 1522 1523 For example, if you want to pass:: 1524 1525 [3, 7, 9, 15] 1526 1527 You can create an argument like this:: 1528 1529 ListOf(Integer()) 1530 1531 The serialized form of the entire list is subject to the limit imposed by 1532 L{MAX_VALUE_LENGTH}. List elements are represented as 16-bit length 1533 prefixed strings. The argument type passed to the L{ListOf} initializer is 1534 responsible for producing the serialized form of each element. 1535 1536 @ivar elementType: The L{Argument} instance used to encode and decode list 1537 elements (note, not an arbitrary L{IArgumentType} implementation: 1538 arguments must be implemented using only the C{fromString} and 1539 C{toString} methods, not the C{fromBox} and C{toBox} methods). 1540 1541 @param optional: a boolean indicating whether this argument can be 1542 omitted in the protocol. 1543 1544 @since: 10.0 1545 """ 1546 1547 def __init__(self, elementType, optional=False): 1548 self.elementType = elementType 1549 Argument.__init__(self, optional) 1550 1551 def fromString(self, inString): 1552 """ 1553 Convert the serialized form of a list of instances of some type back 1554 into that list. 1555 """ 1556 strings = [] 1557 parser = Int16StringReceiver() 1558 parser.stringReceived = strings.append 1559 parser.dataReceived(inString) 1560 elementFromString = self.elementType.fromString 1561 return [elementFromString(string) for string in strings] 1562 1563 def toString(self, inObject): 1564 """ 1565 Serialize the given list of objects to a single string. 1566 """ 1567 strings = [] 1568 for obj in inObject: 1569 serialized = self.elementType.toString(obj) 1570 strings.append(pack("!H", len(serialized))) 1571 strings.append(serialized) 1572 return b"".join(strings) 1573 1574 1575class AmpList(Argument): 1576 """ 1577 Convert a list of dictionaries into a list of AMP boxes on the wire. 1578 1579 For example, if you want to pass:: 1580 1581 [{'a': 7, 'b': u'hello'}, {'a': 9, 'b': u'goodbye'}] 1582 1583 You might use an AmpList like this in your arguments or response list:: 1584 1585 AmpList([('a', Integer()), 1586 ('b', Unicode())]) 1587 """ 1588 1589 def __init__(self, subargs, optional=False): 1590 """ 1591 Create an AmpList. 1592 1593 @param subargs: a list of 2-tuples of ('name', argument) describing the 1594 schema of the dictionaries in the sequence of amp boxes. 1595 @type subargs: A C{list} of (C{bytes}, L{Argument}) tuples. 1596 1597 @param optional: a boolean indicating whether this argument can be 1598 omitted in the protocol. 1599 """ 1600 assert all(isinstance(name, bytes) for name, _ in subargs), ( 1601 "AmpList should be defined with a list of (name, argument) " 1602 "tuples where `name' is a byte string, got: %r" % (subargs,) 1603 ) 1604 self.subargs = subargs 1605 Argument.__init__(self, optional) 1606 1607 def fromStringProto(self, inString, proto): 1608 boxes = parseString(inString) 1609 values = [_stringsToObjects(box, self.subargs, proto) for box in boxes] 1610 return values 1611 1612 def toStringProto(self, inObject, proto): 1613 return b"".join( 1614 [ 1615 _objectsToStrings(objects, self.subargs, Box(), proto).serialize() 1616 for objects in inObject 1617 ] 1618 ) 1619 1620 1621class Descriptor(Integer): 1622 """ 1623 Encode and decode file descriptors for exchange over a UNIX domain socket. 1624 1625 This argument type requires an AMP connection set up over an 1626 L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>} provider (for 1627 example, the kind of connection created by 1628 L{IReactorUNIX.connectUNIX<twisted.internet.interfaces.IReactorUNIX.connectUNIX>} 1629 and L{UNIXClientEndpoint<twisted.internet.endpoints.UNIXClientEndpoint>}). 1630 1631 There is no correspondence between the integer value of the file descriptor 1632 on the sending and receiving sides, therefore an alternate approach is taken 1633 to matching up received descriptors with particular L{Descriptor} 1634 parameters. The argument is encoded to an ordinal (unique per connection) 1635 for inclusion in the AMP command or response box. The descriptor itself is 1636 sent using 1637 L{IUNIXTransport.sendFileDescriptor<twisted.internet.interfaces.IUNIXTransport.sendFileDescriptor>}. 1638 The receiver uses the order in which file descriptors are received and the 1639 ordinal value to come up with the received copy of the descriptor. 1640 """ 1641 1642 def fromStringProto(self, inString, proto): 1643 """ 1644 Take a unique identifier associated with a file descriptor which must 1645 have been received by now and use it to look up that descriptor in a 1646 dictionary where they are kept. 1647 1648 @param inString: The base representation (as a byte string) of an 1649 ordinal indicating which file descriptor corresponds to this usage 1650 of this argument. 1651 @type inString: C{str} 1652 1653 @param proto: The protocol used to receive this descriptor. This 1654 protocol must be connected via a transport providing 1655 L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}. 1656 @type proto: L{BinaryBoxProtocol} 1657 1658 @return: The file descriptor represented by C{inString}. 1659 @rtype: C{int} 1660 """ 1661 return proto._getDescriptor(int(inString)) 1662 1663 def toStringProto(self, inObject, proto): 1664 """ 1665 Send C{inObject}, an integer file descriptor, over C{proto}'s connection 1666 and return a unique identifier which will allow the receiver to 1667 associate the file descriptor with this argument. 1668 1669 @param inObject: A file descriptor to duplicate over an AMP connection 1670 as the value for this argument. 1671 @type inObject: C{int} 1672 1673 @param proto: The protocol which will be used to send this descriptor. 1674 This protocol must be connected via a transport providing 1675 L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}. 1676 1677 @return: A byte string which can be used by the receiver to reconstruct 1678 the file descriptor. 1679 @rtype: C{bytes} 1680 """ 1681 identifier = proto._sendFileDescriptor(inObject) 1682 outString = Integer.toStringProto(self, identifier, proto) 1683 return outString 1684 1685 1686class _CommandMeta(type): 1687 """ 1688 Metaclass hack to establish reverse-mappings for 'errors' and 1689 'fatalErrors' as class vars. 1690 """ 1691 1692 def __new__(cls, name, bases, attrs): 1693 reverseErrors = attrs["reverseErrors"] = {} 1694 er = attrs["allErrors"] = {} 1695 if "commandName" not in attrs: 1696 attrs["commandName"] = name.encode("ascii") 1697 newtype = type.__new__(cls, name, bases, attrs) 1698 1699 if not isinstance(newtype.commandName, bytes): 1700 raise TypeError( 1701 "Command names must be byte strings, got: {!r}".format( 1702 newtype.commandName 1703 ) 1704 ) 1705 for name, _ in newtype.arguments: 1706 if not isinstance(name, bytes): 1707 raise TypeError(f"Argument names must be byte strings, got: {name!r}") 1708 for name, _ in newtype.response: 1709 if not isinstance(name, bytes): 1710 raise TypeError(f"Response names must be byte strings, got: {name!r}") 1711 1712 errors: Dict[Type[Exception], bytes] = {} 1713 fatalErrors: Dict[Type[Exception], bytes] = {} 1714 accumulateClassDict(newtype, "errors", errors) 1715 accumulateClassDict(newtype, "fatalErrors", fatalErrors) 1716 1717 if not isinstance(newtype.errors, dict): 1718 newtype.errors = dict(newtype.errors) 1719 if not isinstance(newtype.fatalErrors, dict): 1720 newtype.fatalErrors = dict(newtype.fatalErrors) 1721 1722 for v, k in errors.items(): 1723 reverseErrors[k] = v 1724 er[v] = k 1725 for v, k in fatalErrors.items(): 1726 reverseErrors[k] = v 1727 er[v] = k 1728 1729 for _, name in newtype.errors.items(): 1730 if not isinstance(name, bytes): 1731 raise TypeError(f"Error names must be byte strings, got: {name!r}") 1732 for _, name in newtype.fatalErrors.items(): 1733 if not isinstance(name, bytes): 1734 raise TypeError( 1735 f"Fatal error names must be byte strings, got: {name!r}" 1736 ) 1737 1738 return newtype 1739 1740 1741class Command(metaclass=_CommandMeta): 1742 """ 1743 Subclass me to specify an AMP Command. 1744 1745 @cvar arguments: A list of 2-tuples of (name, Argument-subclass-instance), 1746 specifying the names and values of the parameters which are required for 1747 this command. 1748 1749 @cvar response: A list like L{arguments}, but instead used for the return 1750 value. 1751 1752 @cvar errors: A mapping of subclasses of L{Exception} to wire-protocol tags 1753 for errors represented as L{str}s. Responders which raise keys from 1754 this dictionary will have the error translated to the corresponding tag 1755 on the wire. 1756 Invokers which receive Deferreds from invoking this command with 1757 L{BoxDispatcher.callRemote} will potentially receive Failures with keys 1758 from this mapping as their value. 1759 This mapping is inherited; if you declare a command which handles 1760 C{FooError} as 'FOO_ERROR', then subclass it and specify C{BarError} as 1761 'BAR_ERROR', responders to the subclass may raise either C{FooError} or 1762 C{BarError}, and invokers must be able to deal with either of those 1763 exceptions. 1764 1765 @cvar fatalErrors: like 'errors', but errors in this list will always 1766 terminate the connection, despite being of a recognizable error type. 1767 1768 @cvar commandType: The type of Box used to issue commands; useful only for 1769 protocol-modifying behavior like startTLS or protocol switching. Defaults 1770 to a plain vanilla L{Box}. 1771 1772 @cvar responseType: The type of Box used to respond to this command; only 1773 useful for protocol-modifying behavior like startTLS or protocol switching. 1774 Defaults to a plain vanilla L{Box}. 1775 1776 @ivar requiresAnswer: a boolean; defaults to True. Set it to False on your 1777 subclass if you want callRemote to return None. Note: this is a hint only 1778 to the client side of the protocol. The return-type of a command responder 1779 method must always be a dictionary adhering to the contract specified by 1780 L{response}, because clients are always free to request a response if they 1781 want one. 1782 """ 1783 1784 arguments: List[Tuple[bytes, Argument]] = [] 1785 response: List[Tuple[bytes, Argument]] = [] 1786 extra: List[Any] = [] 1787 errors: Dict[Type[Exception], bytes] = {} 1788 fatalErrors: Dict[Type[Exception], bytes] = {} 1789 1790 commandType: "Union[Type[Command], Type[Box]]" = Box 1791 responseType: Type[AmpBox] = Box 1792 1793 requiresAnswer = True 1794 1795 def __init__(self, **kw): 1796 """ 1797 Create an instance of this command with specified values for its 1798 parameters. 1799 1800 In Python 3, keyword arguments MUST be Unicode/native strings whereas 1801 in Python 2 they could be either byte strings or Unicode strings. 1802 1803 A L{Command}'s arguments are defined in its schema using C{bytes} 1804 names. The values for those arguments are plucked from the keyword 1805 arguments using the name returned from L{_wireNameToPythonIdentifier}. 1806 In other words, keyword arguments should be named using the 1807 Python-side equivalent of the on-wire (C{bytes}) name. 1808 1809 @param kw: a dict containing an appropriate value for each name 1810 specified in the L{arguments} attribute of my class. 1811 1812 @raise InvalidSignature: if you forgot any required arguments. 1813 """ 1814 self.structured = kw 1815 forgotten = [] 1816 for name, arg in self.arguments: 1817 pythonName = _wireNameToPythonIdentifier(name) 1818 if pythonName not in self.structured and not arg.optional: 1819 forgotten.append(pythonName) 1820 if forgotten: 1821 raise InvalidSignature( 1822 "forgot {} for {}".format(", ".join(forgotten), self.commandName) 1823 ) 1824 forgotten = [] 1825 1826 @classmethod 1827 def makeResponse(cls, objects, proto): 1828 """ 1829 Serialize a mapping of arguments using this L{Command}'s 1830 response schema. 1831 1832 @param objects: a dict with keys matching the names specified in 1833 self.response, having values of the types that the Argument objects in 1834 self.response can format. 1835 1836 @param proto: an L{AMP}. 1837 1838 @return: an L{AmpBox}. 1839 """ 1840 try: 1841 responseType = cls.responseType() 1842 except BaseException: 1843 return fail() 1844 return _objectsToStrings(objects, cls.response, responseType, proto) 1845 1846 @classmethod 1847 def makeArguments(cls, objects, proto): 1848 """ 1849 Serialize a mapping of arguments using this L{Command}'s 1850 argument schema. 1851 1852 @param objects: a dict with keys similar to the names specified in 1853 self.arguments, having values of the types that the Argument objects in 1854 self.arguments can parse. 1855 1856 @param proto: an L{AMP}. 1857 1858 @return: An instance of this L{Command}'s C{commandType}. 1859 """ 1860 allowedNames = set() 1861 for (argName, ignored) in cls.arguments: 1862 allowedNames.add(_wireNameToPythonIdentifier(argName)) 1863 1864 for intendedArg in objects: 1865 if intendedArg not in allowedNames: 1866 raise InvalidSignature(f"{intendedArg} is not a valid argument") 1867 return _objectsToStrings(objects, cls.arguments, cls.commandType(), proto) 1868 1869 @classmethod 1870 def parseResponse(cls, box, protocol): 1871 """ 1872 Parse a mapping of serialized arguments using this 1873 L{Command}'s response schema. 1874 1875 @param box: A mapping of response-argument names to the 1876 serialized forms of those arguments. 1877 @param protocol: The L{AMP} protocol. 1878 1879 @return: A mapping of response-argument names to the parsed 1880 forms. 1881 """ 1882 return _stringsToObjects(box, cls.response, protocol) 1883 1884 @classmethod 1885 def parseArguments(cls, box, protocol): 1886 """ 1887 Parse a mapping of serialized arguments using this 1888 L{Command}'s argument schema. 1889 1890 @param box: A mapping of argument names to the seralized forms 1891 of those arguments. 1892 @param protocol: The L{AMP} protocol. 1893 1894 @return: A mapping of argument names to the parsed forms. 1895 """ 1896 return _stringsToObjects(box, cls.arguments, protocol) 1897 1898 @classmethod 1899 def responder(cls, methodfunc): 1900 """ 1901 Declare a method to be a responder for a particular command. 1902 1903 This is a decorator. 1904 1905 Use like so:: 1906 1907 class MyCommand(Command): 1908 arguments = [('a', ...), ('b', ...)] 1909 1910 class MyProto(AMP): 1911 def myFunMethod(self, a, b): 1912 ... 1913 MyCommand.responder(myFunMethod) 1914 1915 Notes: Although decorator syntax is not used within Twisted, this 1916 function returns its argument and is therefore safe to use with 1917 decorator syntax. 1918 1919 This is not thread safe. Don't declare AMP subclasses in other 1920 threads. Don't declare responders outside the scope of AMP subclasses; 1921 the behavior is undefined. 1922 1923 @param methodfunc: A function which will later become a method, which 1924 has a keyword signature compatible with this command's L{arguments} list 1925 and returns a dictionary with a set of keys compatible with this 1926 command's L{response} list. 1927 1928 @return: the methodfunc parameter. 1929 """ 1930 CommandLocator._currentClassCommands.append((cls, methodfunc)) 1931 return methodfunc 1932 1933 # Our only instance method 1934 def _doCommand(self, proto): 1935 """ 1936 Encode and send this Command to the given protocol. 1937 1938 @param proto: an AMP, representing the connection to send to. 1939 1940 @return: a Deferred which will fire or error appropriately when the 1941 other side responds to the command (or error if the connection is lost 1942 before it is responded to). 1943 """ 1944 1945 def _massageError(error): 1946 error.trap(RemoteAmpError) 1947 rje = error.value 1948 errorType = self.reverseErrors.get(rje.errorCode, UnknownRemoteError) 1949 return Failure(errorType(rje.description)) 1950 1951 d = proto._sendBoxCommand( 1952 self.commandName, 1953 self.makeArguments(self.structured, proto), 1954 self.requiresAnswer, 1955 ) 1956 1957 if self.requiresAnswer: 1958 d.addCallback(self.parseResponse, proto) 1959 d.addErrback(_massageError) 1960 1961 return d 1962 1963 1964class _NoCertificate: 1965 """ 1966 This is for peers which don't want to use a local certificate. Used by 1967 AMP because AMP's internal language is all about certificates and this 1968 duck-types in the appropriate place; this API isn't really stable though, 1969 so it's not exposed anywhere public. 1970 1971 For clients, it will use ephemeral DH keys, or whatever the default is for 1972 certificate-less clients in OpenSSL. For servers, it will generate a 1973 temporary self-signed certificate with garbage values in the DN and use 1974 that. 1975 """ 1976 1977 def __init__(self, client): 1978 """ 1979 Create a _NoCertificate which either is or isn't for the client side of 1980 the connection. 1981 1982 @param client: True if we are a client and should truly have no 1983 certificate and be anonymous, False if we are a server and actually 1984 have to generate a temporary certificate. 1985 1986 @type client: bool 1987 """ 1988 self.client = client 1989 1990 def options(self, *authorities): 1991 """ 1992 Behaves like L{twisted.internet.ssl.PrivateCertificate.options}(). 1993 """ 1994 if not self.client: 1995 # do some crud with sslverify to generate a temporary self-signed 1996 # certificate. This is SLOOOWWWWW so it is only in the absolute 1997 # worst, most naive case. 1998 1999 # We have to do this because OpenSSL will not let both the server 2000 # and client be anonymous. 2001 sharedDN = DN(CN="TEMPORARY CERTIFICATE") 2002 key = KeyPair.generate() 2003 cr = key.certificateRequest(sharedDN) 2004 sscrd = key.signCertificateRequest(sharedDN, cr, lambda dn: True, 1) 2005 cert = key.newCertificate(sscrd) 2006 return cert.options(*authorities) 2007 options = dict() 2008 if authorities: 2009 options.update( 2010 dict( 2011 verify=True, 2012 requireCertificate=True, 2013 caCerts=[auth.original for auth in authorities], 2014 ) 2015 ) 2016 occo = CertificateOptions(**options) 2017 return occo 2018 2019 2020class _TLSBox(AmpBox): 2021 """ 2022 I am an AmpBox that, upon being sent, initiates a TLS connection. 2023 """ 2024 2025 __slots__: List[str] = [] 2026 2027 def __init__(self): 2028 if ssl is None: 2029 raise RemoteAmpError(b"TLS_ERROR", "TLS not available") 2030 AmpBox.__init__(self) 2031 2032 @property 2033 def certificate(self): 2034 return self.get(b"tls_localCertificate", _NoCertificate(False)) 2035 2036 @property 2037 def verify(self): 2038 return self.get(b"tls_verifyAuthorities", None) 2039 2040 def _sendTo(self, proto): 2041 """ 2042 Send my encoded value to the protocol, then initiate TLS. 2043 """ 2044 ab = AmpBox(self) 2045 for k in [b"tls_localCertificate", b"tls_verifyAuthorities"]: 2046 ab.pop(k, None) 2047 ab._sendTo(proto) 2048 proto._startTLS(self.certificate, self.verify) 2049 2050 2051class _LocalArgument(String): 2052 """ 2053 Local arguments are never actually relayed across the wire. This is just a 2054 shim so that StartTLS can pretend to have some arguments: if arguments 2055 acquire documentation properties, replace this with something nicer later. 2056 """ 2057 2058 def fromBox(self, name, strings, objects, proto): 2059 pass 2060 2061 2062class StartTLS(Command): 2063 """ 2064 Use, or subclass, me to implement a command that starts TLS. 2065 2066 Callers of StartTLS may pass several special arguments, which affect the 2067 TLS negotiation: 2068 2069 - tls_localCertificate: This is a 2070 twisted.internet.ssl.PrivateCertificate which will be used to secure 2071 the side of the connection it is returned on. 2072 2073 - tls_verifyAuthorities: This is a list of 2074 twisted.internet.ssl.Certificate objects that will be used as the 2075 certificate authorities to verify our peer's certificate. 2076 2077 Each of those special parameters may also be present as a key in the 2078 response dictionary. 2079 """ 2080 2081 arguments = [ 2082 (b"tls_localCertificate", _LocalArgument(optional=True)), 2083 (b"tls_verifyAuthorities", _LocalArgument(optional=True)), 2084 ] 2085 2086 response = [ 2087 (b"tls_localCertificate", _LocalArgument(optional=True)), 2088 (b"tls_verifyAuthorities", _LocalArgument(optional=True)), 2089 ] 2090 2091 responseType = _TLSBox 2092 2093 def __init__(self, *, tls_localCertificate=None, tls_verifyAuthorities=None, **kw): 2094 """ 2095 Create a StartTLS command. (This is private. Use AMP.callRemote.) 2096 2097 @param tls_localCertificate: the PrivateCertificate object to use to 2098 secure the connection. If it's L{None}, or unspecified, an ephemeral DH 2099 key is used instead. 2100 2101 @param tls_verifyAuthorities: a list of Certificate objects which 2102 represent root certificates to verify our peer with. 2103 """ 2104 if ssl is None: 2105 raise RuntimeError("TLS not available.") 2106 self.certificate = ( 2107 _NoCertificate(True) 2108 if tls_localCertificate is None 2109 else tls_localCertificate 2110 ) 2111 self.authorities = tls_verifyAuthorities 2112 Command.__init__(self, **kw) 2113 2114 def _doCommand(self, proto): 2115 """ 2116 When a StartTLS command is sent, prepare to start TLS, but don't actually 2117 do it; wait for the acknowledgement, then initiate the TLS handshake. 2118 """ 2119 d = Command._doCommand(self, proto) 2120 proto._prepareTLS(self.certificate, self.authorities) 2121 # XXX before we get back to user code we are going to start TLS... 2122 2123 def actuallystart(response): 2124 proto._startTLS(self.certificate, self.authorities) 2125 return response 2126 2127 d.addCallback(actuallystart) 2128 return d 2129 2130 2131class ProtocolSwitchCommand(Command): 2132 """ 2133 Use this command to switch from something Amp-derived to a different 2134 protocol mid-connection. This can be useful to use amp as the 2135 connection-startup negotiation phase. Since TLS is a different layer 2136 entirely, you can use Amp to negotiate the security parameters of your 2137 connection, then switch to a different protocol, and the connection will 2138 remain secured. 2139 """ 2140 2141 def __init__(self, _protoToSwitchToFactory, **kw): 2142 """ 2143 Create a ProtocolSwitchCommand. 2144 2145 @param _protoToSwitchToFactory: a ProtocolFactory which will generate 2146 the Protocol to switch to. 2147 2148 @param kw: Keyword arguments, encoded and handled normally as 2149 L{Command} would. 2150 """ 2151 2152 self.protoToSwitchToFactory = _protoToSwitchToFactory 2153 super().__init__(**kw) 2154 2155 @classmethod 2156 def makeResponse(cls, innerProto, proto): 2157 return _SwitchBox(innerProto) 2158 2159 def _doCommand(self, proto): 2160 """ 2161 When we emit a ProtocolSwitchCommand, lock the protocol, but don't actually 2162 switch to the new protocol unless an acknowledgement is received. If 2163 an error is received, switch back. 2164 """ 2165 d = super()._doCommand(proto) 2166 proto._lockForSwitch() 2167 2168 def switchNow(ign): 2169 innerProto = self.protoToSwitchToFactory.buildProtocol( 2170 proto.transport.getPeer() 2171 ) 2172 proto._switchTo(innerProto, self.protoToSwitchToFactory) 2173 return ign 2174 2175 def handle(ign): 2176 proto._unlockFromSwitch() 2177 self.protoToSwitchToFactory.clientConnectionFailed( 2178 None, Failure(CONNECTION_LOST) 2179 ) 2180 return ign 2181 2182 return d.addCallbacks(switchNow, handle) 2183 2184 2185@implementer(IFileDescriptorReceiver) 2186class _DescriptorExchanger: 2187 """ 2188 L{_DescriptorExchanger} is a mixin for L{BinaryBoxProtocol} which adds 2189 support for receiving file descriptors, a feature offered by 2190 L{IUNIXTransport<twisted.internet.interfaces.IUNIXTransport>}. 2191 2192 @ivar _descriptors: Temporary storage for all file descriptors received. 2193 Values in this dictionary are the file descriptors (as integers). Keys 2194 in this dictionary are ordinals giving the order in which each 2195 descriptor was received. The ordering information is used to allow 2196 L{Descriptor} to determine which is the correct descriptor for any 2197 particular usage of that argument type. 2198 @type _descriptors: C{dict} 2199 2200 @ivar _sendingDescriptorCounter: A no-argument callable which returns the 2201 ordinals, starting from 0. This is used to construct values for 2202 C{_sendFileDescriptor}. 2203 2204 @ivar _receivingDescriptorCounter: A no-argument callable which returns the 2205 ordinals, starting from 0. This is used to construct values for 2206 C{fileDescriptorReceived}. 2207 """ 2208 2209 def __init__(self): 2210 self._descriptors = {} 2211 self._getDescriptor = self._descriptors.pop 2212 self._sendingDescriptorCounter = partial(next, count()) 2213 self._receivingDescriptorCounter = partial(next, count()) 2214 2215 def _sendFileDescriptor(self, descriptor): 2216 """ 2217 Assign and return the next ordinal to the given descriptor after sending 2218 the descriptor over this protocol's transport. 2219 """ 2220 self.transport.sendFileDescriptor(descriptor) 2221 return self._sendingDescriptorCounter() 2222 2223 def fileDescriptorReceived(self, descriptor): 2224 """ 2225 Collect received file descriptors to be claimed later by L{Descriptor}. 2226 2227 @param descriptor: The received file descriptor. 2228 @type descriptor: C{int} 2229 """ 2230 self._descriptors[self._receivingDescriptorCounter()] = descriptor 2231 2232 2233@implementer(IBoxSender) 2234class BinaryBoxProtocol( 2235 StatefulStringProtocol, Int16StringReceiver, _DescriptorExchanger 2236): 2237 """ 2238 A protocol for receiving L{AmpBox}es - key/value pairs - via length-prefixed 2239 strings. A box is composed of: 2240 2241 - any number of key-value pairs, described by: 2242 - a 2-byte network-endian packed key length (of which the first 2243 byte must be null, and the second must be non-null: i.e. the 2244 value of the length must be 1-255) 2245 - a key, comprised of that many bytes 2246 - a 2-byte network-endian unsigned value length (up to the maximum 2247 of 65535) 2248 - a value, comprised of that many bytes 2249 - 2 null bytes 2250 2251 In other words, an even number of strings prefixed with packed unsigned 2252 16-bit integers, and then a 0-length string to indicate the end of the box. 2253 2254 This protocol also implements 2 extra private bits of functionality related 2255 to the byte boundaries between messages; it can start TLS between two given 2256 boxes or switch to an entirely different protocol. However, due to some 2257 tricky elements of the implementation, the public interface to this 2258 functionality is L{ProtocolSwitchCommand} and L{StartTLS}. 2259 2260 @ivar _keyLengthLimitExceeded: A flag which is only true when the 2261 connection is being closed because a key length prefix which was longer 2262 than allowed by the protocol was received. 2263 2264 @ivar boxReceiver: an L{IBoxReceiver} provider, whose 2265 L{IBoxReceiver.ampBoxReceived} method will be invoked for each 2266 L{AmpBox} that is received. 2267 """ 2268 2269 _justStartedTLS = False 2270 _startingTLSBuffer = None 2271 _locked = False 2272 _currentKey = None 2273 _currentBox = None 2274 2275 _keyLengthLimitExceeded = False 2276 2277 hostCertificate = None 2278 noPeerCertificate = False # for tests 2279 innerProtocol: Optional[Protocol] = None 2280 innerProtocolClientFactory = None 2281 2282 def __init__(self, boxReceiver): 2283 _DescriptorExchanger.__init__(self) 2284 self.boxReceiver = boxReceiver 2285 2286 def _switchTo(self, newProto, clientFactory=None): 2287 """ 2288 Switch this BinaryBoxProtocol's transport to a new protocol. You need 2289 to do this 'simultaneously' on both ends of a connection; the easiest 2290 way to do this is to use a subclass of ProtocolSwitchCommand. 2291 2292 @param newProto: the new protocol instance to switch to. 2293 2294 @param clientFactory: the ClientFactory to send the 2295 L{twisted.internet.protocol.ClientFactory.clientConnectionLost} 2296 notification to. 2297 """ 2298 # All the data that Int16Receiver has not yet dealt with belongs to our 2299 # new protocol: luckily it's keeping that in a handy (although 2300 # ostensibly internal) variable for us: 2301 newProtoData = self.recvd 2302 # We're quite possibly in the middle of a 'dataReceived' loop in 2303 # Int16StringReceiver: let's make sure that the next iteration, the 2304 # loop will break and not attempt to look at something that isn't a 2305 # length prefix. 2306 self.recvd = "" 2307 # Finally, do the actual work of setting up the protocol and delivering 2308 # its first chunk of data, if one is available. 2309 self.innerProtocol = newProto 2310 self.innerProtocolClientFactory = clientFactory 2311 newProto.makeConnection(self.transport) 2312 if newProtoData: 2313 newProto.dataReceived(newProtoData) 2314 2315 def sendBox(self, box): 2316 """ 2317 Send a amp.Box to my peer. 2318 2319 Note: transport.write is never called outside of this method. 2320 2321 @param box: an AmpBox. 2322 2323 @raise ProtocolSwitched: if the protocol has previously been switched. 2324 2325 @raise ConnectionLost: if the connection has previously been lost. 2326 """ 2327 if self._locked: 2328 raise ProtocolSwitched( 2329 "This connection has switched: no AMP traffic allowed." 2330 ) 2331 if self.transport is None: 2332 raise ConnectionLost() 2333 if self._startingTLSBuffer is not None: 2334 self._startingTLSBuffer.append(box) 2335 else: 2336 self.transport.write(box.serialize()) 2337 2338 def makeConnection(self, transport): 2339 """ 2340 Notify L{boxReceiver} that it is about to receive boxes from this 2341 protocol by invoking L{IBoxReceiver.startReceivingBoxes}. 2342 """ 2343 self.transport = transport 2344 self.boxReceiver.startReceivingBoxes(self) 2345 self.connectionMade() 2346 2347 def dataReceived(self, data): 2348 """ 2349 Either parse incoming data as L{AmpBox}es or relay it to our nested 2350 protocol. 2351 """ 2352 if self._justStartedTLS: 2353 self._justStartedTLS = False 2354 # If we already have an inner protocol, then we don't deliver data to 2355 # the protocol parser any more; we just hand it off. 2356 if self.innerProtocol is not None: 2357 self.innerProtocol.dataReceived(data) 2358 return 2359 return Int16StringReceiver.dataReceived(self, data) 2360 2361 def connectionLost(self, reason): 2362 """ 2363 The connection was lost; notify any nested protocol. 2364 """ 2365 if self.innerProtocol is not None: 2366 self.innerProtocol.connectionLost(reason) 2367 if self.innerProtocolClientFactory is not None: 2368 self.innerProtocolClientFactory.clientConnectionLost(None, reason) 2369 if self._keyLengthLimitExceeded: 2370 failReason = Failure(TooLong(True, False, None, None)) 2371 elif reason.check(ConnectionClosed) and self._justStartedTLS: 2372 # We just started TLS and haven't received any data. This means 2373 # the other connection didn't like our cert (although they may not 2374 # have told us why - later Twisted should make 'reason' into a TLS 2375 # error.) 2376 failReason = PeerVerifyError( 2377 "Peer rejected our certificate for an unknown reason." 2378 ) 2379 else: 2380 failReason = reason 2381 self.boxReceiver.stopReceivingBoxes(failReason) 2382 2383 # The longest key allowed 2384 _MAX_KEY_LENGTH = 255 2385 2386 # The longest value allowed (this is somewhat redundant, as longer values 2387 # cannot be encoded - ah well). 2388 _MAX_VALUE_LENGTH = 65535 2389 2390 # The first thing received is a key. 2391 MAX_LENGTH = _MAX_KEY_LENGTH 2392 2393 def proto_init(self, string): 2394 """ 2395 String received in the 'init' state. 2396 """ 2397 self._currentBox = AmpBox() 2398 return self.proto_key(string) 2399 2400 def proto_key(self, string): 2401 """ 2402 String received in the 'key' state. If the key is empty, a complete 2403 box has been received. 2404 """ 2405 if string: 2406 self._currentKey = string 2407 self.MAX_LENGTH = self._MAX_VALUE_LENGTH 2408 return "value" 2409 else: 2410 self.boxReceiver.ampBoxReceived(self._currentBox) 2411 self._currentBox = None 2412 return "init" 2413 2414 def proto_value(self, string): 2415 """ 2416 String received in the 'value' state. 2417 """ 2418 self._currentBox[self._currentKey] = string 2419 self._currentKey = None 2420 self.MAX_LENGTH = self._MAX_KEY_LENGTH 2421 return "key" 2422 2423 def lengthLimitExceeded(self, length): 2424 """ 2425 The key length limit was exceeded. Disconnect the transport and make 2426 sure a meaningful exception is reported. 2427 """ 2428 self._keyLengthLimitExceeded = True 2429 self.transport.loseConnection() 2430 2431 def _lockForSwitch(self): 2432 """ 2433 Lock this binary protocol so that no further boxes may be sent. This 2434 is used when sending a request to switch underlying protocols. You 2435 probably want to subclass ProtocolSwitchCommand rather than calling 2436 this directly. 2437 """ 2438 self._locked = True 2439 2440 def _unlockFromSwitch(self): 2441 """ 2442 Unlock this locked binary protocol so that further boxes may be sent 2443 again. This is used after an attempt to switch protocols has failed 2444 for some reason. 2445 """ 2446 if self.innerProtocol is not None: 2447 raise ProtocolSwitched("Protocol already switched. Cannot unlock.") 2448 self._locked = False 2449 2450 def _prepareTLS(self, certificate, verifyAuthorities): 2451 """ 2452 Used by StartTLSCommand to put us into the state where we don't 2453 actually send things that get sent, instead we buffer them. see 2454 L{_sendBoxCommand}. 2455 """ 2456 self._startingTLSBuffer = [] 2457 if self.hostCertificate is not None: 2458 raise OnlyOneTLS( 2459 "Previously authenticated connection between %s and %s " 2460 "is trying to re-establish as %s" 2461 % ( 2462 self.hostCertificate, 2463 self.peerCertificate, 2464 (certificate, verifyAuthorities), 2465 ) 2466 ) 2467 2468 def _startTLS(self, certificate, verifyAuthorities): 2469 """ 2470 Used by TLSBox to initiate the SSL handshake. 2471 2472 @param certificate: a L{twisted.internet.ssl.PrivateCertificate} for 2473 use locally. 2474 2475 @param verifyAuthorities: L{twisted.internet.ssl.Certificate} instances 2476 representing certificate authorities which will verify our peer. 2477 """ 2478 self.hostCertificate = certificate 2479 self._justStartedTLS = True 2480 if verifyAuthorities is None: 2481 verifyAuthorities = () 2482 self.transport.startTLS(certificate.options(*verifyAuthorities)) 2483 stlsb = self._startingTLSBuffer 2484 if stlsb is not None: 2485 self._startingTLSBuffer = None 2486 for box in stlsb: 2487 self.sendBox(box) 2488 2489 @property 2490 def peerCertificate(self): 2491 if self.noPeerCertificate: 2492 return None 2493 return Certificate.peerFromTransport(self.transport) 2494 2495 def unhandledError(self, failure): 2496 """ 2497 The buck stops here. This error was completely unhandled, time to 2498 terminate the connection. 2499 """ 2500 log.err( 2501 failure, 2502 "Amp server or network failure unhandled by client application. " 2503 "Dropping connection! To avoid, add errbacks to ALL remote " 2504 "commands!", 2505 ) 2506 if self.transport is not None: 2507 self.transport.loseConnection() 2508 2509 def _defaultStartTLSResponder(self): 2510 """ 2511 The default TLS responder doesn't specify any certificate or anything. 2512 2513 From a security perspective, it's little better than a plain-text 2514 connection - but it is still a *bit* better, so it's included for 2515 convenience. 2516 2517 You probably want to override this by providing your own StartTLS.responder. 2518 """ 2519 return {} 2520 2521 StartTLS.responder(_defaultStartTLSResponder) 2522 2523 2524class AMP(BinaryBoxProtocol, BoxDispatcher, CommandLocator, SimpleStringLocator): 2525 """ 2526 This protocol is an AMP connection. See the module docstring for protocol 2527 details. 2528 """ 2529 2530 _ampInitialized = False 2531 2532 def __init__(self, boxReceiver=None, locator=None): 2533 # For backwards compatibility. When AMP did not separate parsing logic 2534 # (L{BinaryBoxProtocol}), request-response logic (L{BoxDispatcher}) and 2535 # command routing (L{CommandLocator}), it did not have a constructor. 2536 # Now it does, so old subclasses might have defined their own that did 2537 # not upcall. If this flag isn't set, we'll call the constructor in 2538 # makeConnection before anything actually happens. 2539 self._ampInitialized = True 2540 if boxReceiver is None: 2541 boxReceiver = self 2542 if locator is None: 2543 locator = self 2544 BoxDispatcher.__init__(self, locator) 2545 BinaryBoxProtocol.__init__(self, boxReceiver) 2546 2547 def locateResponder(self, name): 2548 """ 2549 Unify the implementations of L{CommandLocator} and 2550 L{SimpleStringLocator} to perform both kinds of dispatch, preferring 2551 L{CommandLocator}. 2552 2553 @type name: C{bytes} 2554 """ 2555 firstResponder = CommandLocator.locateResponder(self, name) 2556 if firstResponder is not None: 2557 return firstResponder 2558 secondResponder = SimpleStringLocator.locateResponder(self, name) 2559 return secondResponder 2560 2561 def __repr__(self) -> str: 2562 """ 2563 A verbose string representation which gives us information about this 2564 AMP connection. 2565 """ 2566 if self.innerProtocol is not None: 2567 innerRepr = f" inner {self.innerProtocol!r}" 2568 else: 2569 innerRepr = "" 2570 return f"<{self.__class__.__name__}{innerRepr} at 0x{id(self):x}>" 2571 2572 def makeConnection(self, transport): 2573 """ 2574 Emit a helpful log message when the connection is made. 2575 """ 2576 if not self._ampInitialized: 2577 # See comment in the constructor re: backward compatibility. I 2578 # should probably emit a deprecation warning here. 2579 AMP.__init__(self) 2580 # Save these so we can emit a similar log message in L{connectionLost}. 2581 self._transportPeer = transport.getPeer() 2582 self._transportHost = transport.getHost() 2583 log.msg( 2584 "%s connection established (HOST:%s PEER:%s)" 2585 % (self.__class__.__name__, self._transportHost, self._transportPeer) 2586 ) 2587 BinaryBoxProtocol.makeConnection(self, transport) 2588 2589 def connectionLost(self, reason): 2590 """ 2591 Emit a helpful log message when the connection is lost. 2592 """ 2593 log.msg( 2594 "%s connection lost (HOST:%s PEER:%s)" 2595 % (self.__class__.__name__, self._transportHost, self._transportPeer) 2596 ) 2597 BinaryBoxProtocol.connectionLost(self, reason) 2598 self.transport = None 2599 2600 2601class _ParserHelper: 2602 """ 2603 A box receiver which records all boxes received. 2604 """ 2605 2606 def __init__(self): 2607 self.boxes = [] 2608 2609 def getPeer(self): 2610 return "string" 2611 2612 def getHost(self): 2613 return "string" 2614 2615 disconnecting = False 2616 2617 def startReceivingBoxes(self, sender): 2618 """ 2619 No initialization is required. 2620 """ 2621 2622 def ampBoxReceived(self, box): 2623 self.boxes.append(box) 2624 2625 # Synchronous helpers 2626 @classmethod 2627 def parse(cls, fileObj): 2628 """ 2629 Parse some amp data stored in a file. 2630 2631 @param fileObj: a file-like object. 2632 2633 @return: a list of AmpBoxes encoded in the given file. 2634 """ 2635 parserHelper = cls() 2636 bbp = BinaryBoxProtocol(boxReceiver=parserHelper) 2637 bbp.makeConnection(parserHelper) 2638 bbp.dataReceived(fileObj.read()) 2639 return parserHelper.boxes 2640 2641 @classmethod 2642 def parseString(cls, data): 2643 """ 2644 Parse some amp data stored in a string. 2645 2646 @param data: a str holding some amp-encoded data. 2647 2648 @return: a list of AmpBoxes encoded in the given string. 2649 """ 2650 return cls.parse(BytesIO(data)) 2651 2652 2653parse = _ParserHelper.parse 2654parseString = _ParserHelper.parseString 2655 2656 2657def _stringsToObjects(strings, arglist, proto): 2658 """ 2659 Convert an AmpBox to a dictionary of python objects, converting through a 2660 given arglist. 2661 2662 @param strings: an AmpBox (or dict of strings) 2663 2664 @param arglist: a list of 2-tuples of strings and Argument objects, as 2665 described in L{Command.arguments}. 2666 2667 @param proto: an L{AMP} instance. 2668 2669 @return: the converted dictionary mapping names to argument objects. 2670 """ 2671 objects = {} 2672 myStrings = strings.copy() 2673 for argname, argparser in arglist: 2674 argparser.fromBox(argname, myStrings, objects, proto) 2675 return objects 2676 2677 2678def _objectsToStrings(objects, arglist, strings, proto): 2679 """ 2680 Convert a dictionary of python objects to an AmpBox, converting through a 2681 given arglist. 2682 2683 @param objects: a dict mapping names to python objects 2684 2685 @param arglist: a list of 2-tuples of strings and Argument objects, as 2686 described in L{Command.arguments}. 2687 2688 @param strings: [OUT PARAMETER] An object providing the L{dict} 2689 interface which will be populated with serialized data. 2690 2691 @param proto: an L{AMP} instance. 2692 2693 @return: The converted dictionary mapping names to encoded argument 2694 strings (identical to C{strings}). 2695 """ 2696 myObjects = objects.copy() 2697 for argname, argparser in arglist: 2698 argparser.toBox(argname, strings, myObjects, proto) 2699 return strings 2700 2701 2702class Decimal(Argument): 2703 """ 2704 Encodes C{decimal.Decimal} instances. 2705 2706 There are several ways in which a decimal value might be encoded. 2707 2708 Special values are encoded as special strings:: 2709 2710 - Positive infinity is encoded as C{"Infinity"} 2711 - Negative infinity is encoded as C{"-Infinity"} 2712 - Quiet not-a-number is encoded as either C{"NaN"} or C{"-NaN"} 2713 - Signalling not-a-number is encoded as either C{"sNaN"} or C{"-sNaN"} 2714 2715 Normal values are encoded using the base ten string representation, using 2716 engineering notation to indicate magnitude without precision, and "normal" 2717 digits to indicate precision. For example:: 2718 2719 - C{"1"} represents the value I{1} with precision to one place. 2720 - C{"-1"} represents the value I{-1} with precision to one place. 2721 - C{"1.0"} represents the value I{1} with precision to two places. 2722 - C{"10"} represents the value I{10} with precision to two places. 2723 - C{"1E+2"} represents the value I{10} with precision to one place. 2724 - C{"1E-1"} represents the value I{0.1} with precision to one place. 2725 - C{"1.5E+2"} represents the value I{15} with precision to two places. 2726 2727 U{http://speleotrove.com/decimal/} should be considered the authoritative 2728 specification for the format. 2729 """ 2730 2731 def fromString(self, inString): 2732 inString = nativeString(inString) 2733 return decimal.Decimal(inString) 2734 2735 def toString(self, inObject): 2736 """ 2737 Serialize a C{decimal.Decimal} instance to the specified wire format. 2738 """ 2739 if isinstance(inObject, decimal.Decimal): 2740 # Hopefully decimal.Decimal.__str__ actually does what we want. 2741 return str(inObject).encode("ascii") 2742 raise ValueError("amp.Decimal can only encode instances of decimal.Decimal") 2743 2744 2745class DateTime(Argument): 2746 """ 2747 Encodes C{datetime.datetime} instances. 2748 2749 Wire format: '%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i'. Fields in 2750 order are: year, month, day, hour, minute, second, microsecond, timezone 2751 direction (+ or -), timezone hour, timezone minute. Encoded string is 2752 always exactly 32 characters long. This format is compatible with ISO 8601, 2753 but that does not mean all ISO 8601 dates can be accepted. 2754 2755 Also, note that the datetime module's notion of a "timezone" can be 2756 complex, but the wire format includes only a fixed offset, so the 2757 conversion is not lossless. A lossless transmission of a C{datetime} instance 2758 is not feasible since the receiving end would require a Python interpreter. 2759 2760 @ivar _positions: A sequence of slices giving the positions of various 2761 interesting parts of the wire format. 2762 """ 2763 2764 _positions = [ 2765 slice(0, 4), 2766 slice(5, 7), 2767 slice(8, 10), # year, month, day 2768 slice(11, 13), 2769 slice(14, 16), 2770 slice(17, 19), # hour, minute, second 2771 slice(20, 26), # microsecond 2772 # intentionally skip timezone direction, as it is not an integer 2773 slice(27, 29), 2774 slice(30, 32), # timezone hour, timezone minute 2775 ] 2776 2777 def fromString(self, s): 2778 """ 2779 Parse a string containing a date and time in the wire format into a 2780 C{datetime.datetime} instance. 2781 """ 2782 s = nativeString(s) 2783 2784 if len(s) != 32: 2785 raise ValueError(f"invalid date format {s!r}") 2786 2787 values = [int(s[p]) for p in self._positions] 2788 sign = s[26] 2789 timezone = _FixedOffsetTZInfo.fromSignHoursMinutes(sign, *values[7:]) 2790 values[7:] = [timezone] 2791 return datetime.datetime(*values) 2792 2793 def toString(self, i): 2794 """ 2795 Serialize a C{datetime.datetime} instance to a string in the specified 2796 wire format. 2797 """ 2798 offset = i.utcoffset() 2799 if offset is None: 2800 raise ValueError( 2801 "amp.DateTime cannot serialize naive datetime instances. " 2802 "You may find amp.utc useful." 2803 ) 2804 2805 minutesOffset = (offset.days * 86400 + offset.seconds) // 60 2806 2807 if minutesOffset > 0: 2808 sign = "+" 2809 else: 2810 sign = "-" 2811 2812 # strftime has no way to format the microseconds, or put a ':' in the 2813 # timezone. Surprise! 2814 2815 # Python 3.4 cannot do % interpolation on byte strings so we pack into 2816 # an explicitly Unicode string then encode as ASCII. 2817 packed = "%04i-%02i-%02iT%02i:%02i:%02i.%06i%s%02i:%02i" % ( 2818 i.year, 2819 i.month, 2820 i.day, 2821 i.hour, 2822 i.minute, 2823 i.second, 2824 i.microsecond, 2825 sign, 2826 abs(minutesOffset) // 60, 2827 abs(minutesOffset) % 60, 2828 ) 2829 2830 return packed.encode("ascii") 2831