1 /* 2 * Copyright (c) Facebook, Inc. and its affiliates. 3 * 4 * This source code is licensed under the MIT license found in the 5 * LICENSE file in the root directory of this source tree. 6 * 7 */ 8 9 #pragma once 10 11 #include <quic/QuicConstants.h> 12 #include <quic/api/Observer.h> 13 #include <quic/codec/ConnectionIdAlgo.h> 14 #include <quic/codec/QuicReadCodec.h> 15 #include <quic/codec/QuicWriteCodec.h> 16 #include <quic/codec/Types.h> 17 #include <quic/common/BufAccessor.h> 18 #include <quic/common/WindowedCounter.h> 19 #include <quic/d6d/ProbeSizeRaiser.h> 20 #include <quic/handshake/HandshakeLayer.h> 21 #include <quic/logging/QLogger.h> 22 #include <quic/state/AckStates.h> 23 #include <quic/state/LossState.h> 24 #include <quic/state/OutstandingPacket.h> 25 #include <quic/state/PacketEvent.h> 26 #include <quic/state/PendingPathRateLimiter.h> 27 #include <quic/state/QuicConnectionStats.h> 28 #include <quic/state/QuicStreamManager.h> 29 #include <quic/state/QuicTransportStatsCallback.h> 30 #include <quic/state/StreamData.h> 31 #include <quic/state/TransportSettings.h> 32 33 #include <folly/Optional.h> 34 #include <folly/io/IOBuf.h> 35 #include <folly/io/async/AsyncUDPSocket.h> 36 #include <folly/io/async/DelayedDestruction.h> 37 #include <folly/io/async/HHWheelTimer.h> 38 39 #include <chrono> 40 #include <list> 41 #include <numeric> 42 #include <queue> 43 44 namespace quic { 45 46 struct RecvmmsgStorage { 47 // Storage for the recvmmsg system call. 48 std::vector<struct mmsghdr> msgs; 49 std::vector<struct sockaddr_storage> addrs; 50 std::vector<struct iovec> iovecs; 51 // Buffers we pass to recvmmsg. 52 std::vector<Buf> readBuffers; 53 // Free buffers which were not used in previous iterations. 54 std::vector<Buf> freeBufs; 55 resizeRecvmmsgStorage56 void resize(size_t numPackets) { 57 if (msgs.size() != numPackets) { 58 msgs.resize(numPackets); 59 addrs.resize(numPackets); 60 readBuffers.resize(numPackets); 61 iovecs.resize(numPackets); 62 freeBufs.reserve(numPackets); 63 } 64 } 65 }; 66 67 struct NetworkData { 68 TimePoint receiveTimePoint; 69 std::vector<Buf> packets; 70 size_t totalData{0}; 71 72 NetworkData() = default; NetworkDataNetworkData73 NetworkData(Buf&& buf, const TimePoint& receiveTime) 74 : receiveTimePoint(receiveTime) { 75 if (buf) { 76 totalData = buf->computeChainDataLength(); 77 packets.emplace_back(std::move(buf)); 78 } 79 } 80 moveAllDataNetworkData81 std::unique_ptr<folly::IOBuf> moveAllData() && { 82 std::unique_ptr<folly::IOBuf> buf; 83 for (size_t i = 0; i < packets.size(); ++i) { 84 if (buf) { 85 buf->prependChain(std::move(packets[i])); 86 } else { 87 buf = std::move(packets[i]); 88 } 89 } 90 return buf; 91 } 92 }; 93 94 struct NetworkDataSingle { 95 Buf data; 96 TimePoint receiveTimePoint; 97 size_t totalData{0}; 98 99 NetworkDataSingle() = default; 100 NetworkDataSingleNetworkDataSingle101 NetworkDataSingle( 102 std::unique_ptr<folly::IOBuf> buf, 103 const TimePoint& receiveTime) 104 : data(std::move(buf)), receiveTimePoint(receiveTime) { 105 if (data) { 106 totalData += data->computeChainDataLength(); 107 } 108 } 109 }; 110 111 struct OutstandingsInfo { 112 // Sent packets which have not been acked. These are sorted by PacketNum. 113 std::deque<OutstandingPacket> packets; 114 115 // All PacketEvents of this connection. If a OutstandingPacket doesn't have an 116 // associatedEvent or if it's not in this set, there is no need to process its 117 // frames upon ack or loss. 118 folly::F14FastSet<PacketEvent, PacketEventHash> packetEvents; 119 120 // Number of outstanding packets not including cloned 121 EnumArray<PacketNumberSpace, uint64_t> packetCount{}; 122 123 // Number of packets are clones or cloned. 124 EnumArray<PacketNumberSpace, uint64_t> clonedPacketCount{}; 125 126 // Number of packets currently declared lost. 127 uint64_t declaredLostCount{0}; 128 129 // Number of outstanding inflight DSR packet. That is, when a DSR packet is 130 // declared lost, this counter will be decreased. 131 uint64_t dsrCount{0}; 132 133 // Number of packets outstanding and not declared lost. numOutstandingOutstandingsInfo134 uint64_t numOutstanding() { 135 return packets.size() - declaredLostCount; 136 } 137 138 // Total number of cloned packets. numClonedPacketsOutstandingsInfo139 uint64_t numClonedPackets() { 140 return clonedPacketCount[PacketNumberSpace::Initial] + 141 clonedPacketCount[PacketNumberSpace::Handshake] + 142 clonedPacketCount[PacketNumberSpace::AppData]; 143 } 144 }; 145 146 struct Pacer { 147 virtual ~Pacer() = default; 148 149 /** 150 * API for CongestionController to notify Pacer the latest cwnd value in bytes 151 * and connection RTT so that Pacer can recalculate pacing rates. 152 * Note the third parameter is here for testing purposes. 153 */ 154 virtual void refreshPacingRate( 155 uint64_t cwndBytes, 156 std::chrono::microseconds rtt, 157 TimePoint currentTime = Clock::now()) = 0; 158 159 /** 160 * Set the pacers rate to the given value in Bytes per second 161 */ 162 virtual void setPacingRate(uint64_t rateBps) = 0; 163 164 /** 165 * Set an upper limit on the rate this pacer can use. 166 * - If the pacer is currently using a faster pace, it will be brought down to 167 * maxRateBytesPerSec. 168 * - If refreshPacingRate or setPacingRate are called with a value 169 * greated than maxRateBytesPerSec, maxRateBytesPerSec will be used instead. 170 */ 171 virtual void setMaxPacingRate(uint64_t maxRateBytesPerSec) = 0; 172 173 /** 174 * Resets the pacer, which should have the effect of the next write 175 * happening immediately. 176 */ 177 virtual void reset() = 0; 178 179 /** 180 * Set the factor by which to multiply the RTT before determining the 181 * inter-burst interval. E.g. a numerator of 1 and a denominator of 2 182 * would effectively double the pacing rate. 183 */ 184 virtual void setRttFactor(uint8_t numerator, uint8_t denominator) = 0; 185 186 /** 187 * API for Trnasport to query the interval before next write 188 */ 189 [[nodiscard]] virtual std::chrono::microseconds getTimeUntilNextWrite( 190 TimePoint currentTime = Clock::now()) const = 0; 191 192 /** 193 * API for Transport to query a recalculated batch size based on currentTime 194 * and previously scheduled write time. The batch size is how many packets the 195 * transport can write out per eventbase loop. 196 * 197 * currentTime: The caller is expected to pass in a TimePoint value so that 198 * the Pacer can compensate the timer drift. 199 */ 200 virtual uint64_t updateAndGetWriteBatchSize(TimePoint currentTime) = 0; 201 202 /** 203 * Getter API of the most recent write batch size. 204 */ 205 virtual uint64_t getCachedWriteBatchSize() const = 0; 206 207 virtual void onPacketSent() = 0; 208 virtual void onPacketsLoss() = 0; 209 }; 210 211 struct PacingRate { 212 std::chrono::microseconds interval{0us}; 213 uint64_t burstSize{0}; 214 215 struct Builder { 216 Builder&& setInterval(std::chrono::microseconds interval) &&; 217 Builder&& setBurstSize(uint64_t burstSize) &&; 218 PacingRate build() &&; 219 220 private: 221 std::chrono::microseconds interval_{0us}; 222 uint64_t burstSize_{0}; 223 }; 224 225 private: 226 PacingRate(std::chrono::microseconds interval, uint64_t burstSize); 227 }; 228 229 struct CongestionController { 230 // Helper struct to group multiple lost packets into one event 231 struct LossEvent { 232 folly::Optional<PacketNum> largestLostPacketNum; 233 uint64_t lostBytes; 234 uint32_t lostPackets; 235 const TimePoint lossTime; 236 // The packet sent time of the lost packet with largest packet sent time in 237 // this LossEvent 238 folly::Optional<TimePoint> largestLostSentTime; 239 // The packet sent time of the lost packet with smallest packet sent time in 240 // the LossEvent 241 folly::Optional<TimePoint> smallestLostSentTime; 242 // Whether this LossEvent also indicates persistent congestion 243 bool persistentCongestion; 244 245 explicit LossEvent(TimePoint time = Clock::now()) 246 : lostBytes(0), 247 lostPackets(0), 248 lossTime(time), 249 persistentCongestion(false) {} 250 addLostPacketCongestionController::LossEvent251 void addLostPacket(const OutstandingPacket& packet) { 252 if (std::numeric_limits<uint64_t>::max() - lostBytes < 253 packet.metadata.encodedSize) { 254 throw QuicInternalException( 255 "LossEvent: lostBytes overflow", 256 LocalErrorCode::LOST_BYTES_OVERFLOW); 257 } 258 PacketNum packetNum = packet.packet.header.getPacketSequenceNum(); 259 largestLostPacketNum = 260 std::max(packetNum, largestLostPacketNum.value_or(packetNum)); 261 lostBytes += packet.metadata.encodedSize; 262 lostPackets++; 263 largestLostSentTime = std::max( 264 packet.metadata.time, 265 largestLostSentTime.value_or(packet.metadata.time)); 266 smallestLostSentTime = std::min( 267 packet.metadata.time, 268 smallestLostSentTime.value_or(packet.metadata.time)); 269 } 270 }; 271 272 struct AckEvent { 273 /** 274 * The reason that this is an optional type, is that we construct an 275 * AckEvent first, then go through the acked packets that are still 276 * outstanding, and figure out the largest acked packet along the way. 277 */ 278 folly::Optional<PacketNum> largestAckedPacket; 279 TimePoint largestAckedPacketSentTime; 280 bool largestAckedPacketAppLimited{false}; 281 uint64_t ackedBytes{0}; 282 TimePoint ackTime; 283 TimePoint adjustedAckTime; 284 // The minimal RTT sample among packets acked by this AckEvent. This RTT 285 // includes ack delay. 286 folly::Optional<std::chrono::microseconds> mrttSample; 287 // If this AckEvent came from an implicit ACK rather than a real one. 288 bool implicit{false}; 289 290 struct AckPacket { 291 // Packet sent time when this acked pakcet was first sent. 292 TimePoint sentTime; 293 // Packet's encoded size. 294 uint32_t encodedSize; 295 // LastAckedPacketInfo from this acked packet'r original sent 296 // OutstandingPacket structure. 297 folly::Optional<OutstandingPacket::LastAckedPacketInfo> 298 lastAckedPacketInfo; 299 // Total bytes sent on the connection when the acked packet was first 300 // sent. 301 uint64_t totalBytesSentThen; 302 // Whether this packet was sent when CongestionController is in 303 // app-limited state. 304 bool isAppLimited; 305 306 struct Builder { 307 Builder&& setSentTime(TimePoint sentTimeIn); 308 Builder&& setEncodedSize(uint32_t encodedSizeIn); 309 Builder&& setLastAckedPacketInfo( 310 folly::Optional<OutstandingPacket::LastAckedPacketInfo> 311 lastAckedPacketInfoIn); 312 Builder&& setTotalBytesSentThen(uint64_t totalBytesSentThenIn); 313 Builder&& setAppLimited(bool appLimitedIn); 314 AckPacket build() &&; 315 explicit Builder() = default; 316 317 private: 318 TimePoint sentTime; 319 uint32_t encodedSize{0}; 320 folly::Optional<OutstandingPacket::LastAckedPacketInfo> 321 lastAckedPacketInfo; 322 uint64_t totalBytesSentThen{0}; 323 bool isAppLimited{false}; 324 }; 325 326 private: 327 explicit AckPacket( 328 TimePoint sentTimeIn, 329 uint32_t encodedSizeIn, 330 folly::Optional<OutstandingPacket::LastAckedPacketInfo> 331 lastAckedPacketInfoIn, 332 uint64_t totalBytesSentThenIn, 333 bool isAppLimitedIn); 334 }; 335 336 std::vector<AckPacket> ackedPackets; 337 }; 338 339 virtual ~CongestionController() = default; 340 341 /** 342 * Take bytes out of flight without mutating other states of the controller 343 */ 344 virtual void onRemoveBytesFromInflight(uint64_t) = 0; 345 virtual void onPacketSent(const OutstandingPacket& packet) = 0; 346 virtual void onPacketAckOrLoss( 347 folly::Optional<AckEvent>, 348 folly::Optional<LossEvent>) = 0; 349 350 /** 351 * Return the number of bytes that the congestion controller 352 * will allow you to write. 353 */ 354 virtual uint64_t getWritableBytes() const = 0; 355 356 /** 357 * Return the number of bytes of cwnd of the congestion 358 * controller. 359 */ 360 virtual uint64_t getCongestionWindow() const = 0; 361 /** 362 * Notify congestion controller that the connection has become idle or active 363 * in the sense that there are active non-control streams. 364 * idle: true if the connection has become app-idle, false if the 365 * connection has become not app-idle. 366 * eventTime: the time point when the app-idle state changed. 367 */ 368 virtual void setAppIdle(bool idle, TimePoint eventTime) = 0; 369 370 /** 371 * Notify congestion controller that the connection has become app-limited or 372 * not app-limited. 373 * 374 */ 375 virtual void setAppLimited() = 0; 376 virtual CongestionControlType type() const = 0; 377 378 /** 379 * Set the congestion controller to use only a fraction of the available 380 * bandwidth (best-effort for implementations that support it) 381 * bandwidthUtilizationFactor: 382 * < 1.0 indicates backgrounded flow 383 * = 1.0 indicates normal operation. 384 * > 1.0 maps to =1.0 385 */ 386 virtual void setBandwidthUtilizationFactor( 387 float bandwidthUtilizationFactor) noexcept = 0; 388 389 /** 390 * Whether the congestion controller is making use of all of the available 391 * bandwidth. Returns true if bandwidthUtilizationFactor < 1.0. 392 */ 393 virtual bool isInBackgroundMode() const = 0; 394 395 /** 396 * Whether the congestion controller thinks it's currently in app-limited 397 * state. 398 */ 399 virtual bool isAppLimited() const = 0; 400 401 virtual void getStats(CongestionControllerStats& stats) const = 0; 402 }; 403 404 struct QuicCryptoStream : public QuicStreamLike { 405 ~QuicCryptoStream() override = default; 406 }; 407 408 struct QuicCryptoState { 409 // Stream to exchange the initial cryptographic material. 410 QuicCryptoStream initialStream; 411 412 // Stream to exchange the one rtt key material. 413 QuicCryptoStream handshakeStream; 414 415 // Stream to exchange handshake data encrypted with 1-rtt keys. 416 QuicCryptoStream oneRttStream; 417 }; 418 419 struct ConnectionCloseEvent { 420 TransportErrorCode errorCode; 421 std::string reasonPhrase; 422 PacketNum packetSequenceNum; 423 }; 424 425 struct RstStreamEvent { RstStreamEventRstStreamEvent426 RstStreamEvent(StreamId id, uint64_t offset, ApplicationErrorCode error) 427 : stream(id), byteOffset(offset), errorCode(error) {} 428 429 StreamId stream; 430 uint64_t byteOffset; 431 ApplicationErrorCode errorCode; 432 }; 433 434 using Resets = folly::F14FastMap<StreamId, RstStreamFrame>; 435 436 using FrameList = std::vector<QuicSimpleFrame>; 437 438 class Logger; 439 class CongestionControllerFactory; 440 class LoopDetectorCallback; 441 class PendingPathRateLimiter; 442 443 struct QuicConnectionStateBase : public folly::DelayedDestruction { 444 virtual ~QuicConnectionStateBase() = default; 445 QuicConnectionStateBaseQuicConnectionStateBase446 explicit QuicConnectionStateBase(QuicNodeType type) : nodeType(type) { 447 observers = std::make_shared<ObserverVec>(); 448 } 449 450 // Accessor to output buffer for continuous memory GSO writes 451 BufAccessor* bufAccessor{nullptr}; 452 453 std::unique_ptr<Handshake> handshakeLayer; 454 455 // Crypto stream 456 std::unique_ptr<QuicCryptoState> cryptoState; 457 458 // Connection Congestion controller 459 std::unique_ptr<CongestionController> congestionController; 460 461 // Pacer 462 std::unique_ptr<Pacer> pacer; 463 464 // Congestion Controller factory to create specific impl of cc algorithm 465 std::shared_ptr<CongestionControllerFactory> congestionControllerFactory; 466 467 std::unique_ptr<QuicStreamManager> streamManager; 468 469 // When server receives early data attempt without valid source address token, 470 // server will limit bytes in flight to avoid amplification attack. 471 // This limit should be cleared and set back to max after CFIN is received. 472 folly::Optional<uint32_t> writableBytesLimit; 473 474 std::unique_ptr<PendingPathRateLimiter> pathValidationLimiter; 475 476 // Outstanding packets, packet events, and associated counters wrapped in one 477 // class 478 OutstandingsInfo outstandings; 479 480 // The read codec to decrypt and decode packets. 481 std::unique_ptr<QuicReadCodec> readCodec; 482 483 // Initial header cipher. 484 std::unique_ptr<PacketNumberCipher> initialHeaderCipher; 485 486 // Handshake header cipher. 487 std::unique_ptr<PacketNumberCipher> handshakeWriteHeaderCipher; 488 489 // One rtt write header cipher. 490 std::unique_ptr<PacketNumberCipher> oneRttWriteHeaderCipher; 491 492 // Write cipher for 1-RTT data 493 std::unique_ptr<Aead> oneRttWriteCipher; 494 495 // Write cipher for packets with initial keys. 496 std::unique_ptr<Aead> initialWriteCipher; 497 498 // Write cipher for packets with handshake keys. 499 std::unique_ptr<Aead> handshakeWriteCipher; 500 501 // Time at which the connection started. 502 TimePoint connectionTime; 503 504 // The received active_connection_id_limit transport parameter from the peer. 505 uint64_t peerActiveConnectionIdLimit{0}; 506 507 // The destination connection id used in client's initial packet. 508 folly::Optional<ConnectionId> clientChosenDestConnectionId; 509 510 // The source connection id used in client's initial packet. 511 folly::Optional<ConnectionId> clientConnectionId; 512 513 // The current server chosen connection id. 514 folly::Optional<ConnectionId> serverConnectionId; 515 516 // Connection ids issued by self. 517 std::vector<ConnectionIdData> selfConnectionIds; 518 519 // Connection ids issued by peer - to be used as destination ids. 520 std::vector<ConnectionIdData> peerConnectionIds; 521 522 // ConnectionIdAlgo implementation to encode and decode ConnectionId with 523 // various info, such as routing related info. 524 ConnectionIdAlgo* connIdAlgo{nullptr}; 525 526 // Negotiated version. 527 folly::Optional<QuicVersion> version; 528 529 // Original advertised version. Only meaningful to clients. 530 // TODO: move to client only conn state. 531 folly::Optional<QuicVersion> originalVersion; 532 533 // Original address used by the peer when first establishing the connection. 534 folly::SocketAddress originalPeerAddress; 535 536 // Current peer address. 537 folly::SocketAddress peerAddress; 538 539 // Local address. INADDR_ANY if not set. 540 folly::Optional<folly::SocketAddress> localAddress; 541 542 // Local error on the connection. 543 folly::Optional<std::pair<QuicErrorCode, std::string>> localConnectionError; 544 545 // Error sent on the connection by the peer. 546 folly::Optional<std::pair<QuicErrorCode, std::string>> peerConnectionError; 547 548 // Supported versions in order of preference. Only meaningful to clients. 549 // TODO: move to client only conn state. 550 std::vector<QuicVersion> supportedVersions; 551 552 // The endpoint attempts to create a new self connection id with sequence 553 // number and stateless reset token for itself, and if successful, returns it 554 // and updates the connection's state to ensure its peer can use it. createAndAddNewSelfConnIdQuicConnectionStateBase555 virtual folly::Optional<ConnectionIdData> createAndAddNewSelfConnId() { 556 return folly::none; 557 } 558 559 uint64_t nextSelfConnectionIdSequence{0}; 560 561 // D6D related events 562 struct PendingD6DEvents { 563 // If we should schedule/cancel d6d raise timeout, if it's not 564 // already scheduled/canceled 565 bool scheduleRaiseTimeout{false}; 566 567 // If we should schedule/cancel d6d probe timeout, if it's not 568 // already scheduled/canceled 569 bool scheduleProbeTimeout{false}; 570 571 // To send a d6d probe packet 572 bool sendProbePacket{false}; 573 574 // The delay after which sendD6DProbePacket will be set 575 folly::Optional<std::chrono::milliseconds> sendProbeDelay; 576 }; 577 578 struct PendingEvents { 579 Resets resets; 580 folly::Optional<PathChallengeFrame> pathChallenge; 581 582 FrameList frames; 583 584 // D6D related events 585 PendingD6DEvents d6d; 586 587 std::vector<KnobFrame> knobs; 588 589 // Number of probing packets to send after PTO 590 EnumArray<PacketNumberSpace, uint8_t> numProbePackets{}; 591 anyProbePacketsQuicConnectionStateBase::PendingEvents592 FOLLY_NODISCARD bool anyProbePackets() const { 593 return numProbePackets[PacketNumberSpace::Initial] + 594 numProbePackets[PacketNumberSpace::Handshake] + 595 numProbePackets[PacketNumberSpace::AppData]; 596 } 597 598 // true: schedule timeout if not scheduled 599 // false: cancel scheduled timeout 600 bool schedulePathValidationTimeout{false}; 601 602 // If we should schedule a new Ack timeout, if it's not already scheduled 603 bool scheduleAckTimeout{false}; 604 605 // Whether a connection level window update is due to send 606 bool connWindowUpdate{false}; 607 608 // If there is a pending loss detection alarm update 609 bool setLossDetectionAlarm{false}; 610 611 bool cancelPingTimeout{false}; 612 613 // close transport when the next packet number reaches kMaxPacketNum 614 bool closeTransport{false}; 615 616 // To send a ping frame 617 bool sendPing{false}; 618 619 // Do we need to send data blocked frame when connection is blocked. 620 bool sendDataBlocked{false}; 621 }; 622 623 PendingEvents pendingEvents; 624 625 LossState lossState; 626 627 // This contains the ack and packet number related states for all three 628 // packet number space. 629 AckStates ackStates; 630 631 struct ConnectionFlowControlState { 632 // The size of the connection flow control window. 633 uint64_t windowSize{0}; 634 // The max data we have advertised to the peer. 635 uint64_t advertisedMaxOffset{0}; 636 // The max data the peer has advertised on the connection. 637 // This is set to 0 initially so that we can't send any data until we know 638 // the peer's flow control offset. 639 uint64_t peerAdvertisedMaxOffset{0}; 640 // The sum of the min(read offsets) of all the streams on the conn. 641 uint64_t sumCurReadOffset{0}; 642 // The sum of the max(offset) observed on all the streams on the conn. 643 uint64_t sumMaxObservedOffset{0}; 644 // The sum of write offsets of all the streams, only including the offsets 645 // written on the wire. 646 uint64_t sumCurWriteOffset{0}; 647 // The sum of length of data in all the stream buffers. 648 uint64_t sumCurStreamBufferLen{0}; 649 // The packet number in which we got the last largest max data. 650 folly::Optional<PacketNum> largestMaxOffsetReceived; 651 // The following are advertised by the peer, and are set to zero initially 652 // so that we cannot send any data until we know the peer values. 653 // The initial max stream offset for peer-initiated bidirectional streams. 654 uint64_t peerAdvertisedInitialMaxStreamOffsetBidiLocal{0}; 655 // The initial max stream offset for local-initiated bidirectional streams. 656 uint64_t peerAdvertisedInitialMaxStreamOffsetBidiRemote{0}; 657 // The initial max stream offset for unidirectional streams. 658 uint64_t peerAdvertisedInitialMaxStreamOffsetUni{0}; 659 // Time at which the last flow control update was sent by the transport. 660 folly::Optional<TimePoint> timeOfLastFlowControlUpdate; 661 }; 662 663 // Current state of flow control. 664 ConnectionFlowControlState flowControlState; 665 666 // The outstanding path challenge 667 folly::Optional<PathChallengeFrame> outstandingPathValidation; 668 669 // Settings for transports. 670 TransportSettings transportSettings; 671 672 // Value of the negotiated ack delay exponent. 673 uint64_t peerAckDelayExponent{kDefaultAckDelayExponent}; 674 675 // The value of the peer's min_ack_delay, for creating ACK_FREQUENCY frames. 676 folly::Optional<std::chrono::microseconds> peerMinAckDelay; 677 678 // Idle timeout advertised by the peer. Initially sets it to the maximum value 679 // until the handshake sets the timeout. 680 std::chrono::milliseconds peerIdleTimeout{kMaxIdleTimeout}; 681 682 // The max UDP packet size we will be sending, limited by both the received 683 // max_packet_size in Transport Parameters and PMTU 684 uint64_t udpSendPacketLen{kDefaultUDPSendPacketLen}; 685 686 // Peer-advertised max UDP payload size, stored as an opportunistic value to 687 // use when receiving the forciblySetUdpPayloadSize transport knob param 688 uint64_t peerMaxUdpPayloadSize{kDefaultUDPSendPacketLen}; 689 690 struct PacketSchedulingState { 691 StreamId nextScheduledControlStream{0}; 692 }; 693 694 PacketSchedulingState schedulingState; 695 696 // Logger for this connection. 697 std::shared_ptr<Logger> logger; 698 699 // QLogger for this connection 700 std::shared_ptr<QLogger> qLogger; 701 702 // Track stats for various server events 703 QuicTransportStatsCallback* statsCallback{nullptr}; 704 705 // Meta state of d6d, mostly useful for analytics. D6D can operate without it. 706 struct D6DMetaState { 707 // Cumulative count of acked packets 708 uint64_t totalAckedProbes{0}; 709 710 // Cumulative count of lost packets 711 uint64_t totalLostProbes{0}; 712 713 // Cumulative count of transmitted packets 714 uint64_t totalTxedProbes{0}; 715 716 // Timepoint of when d6d reaches a non-search state 717 // this helps us understand the convergence speed 718 TimePoint timeLastNonSearchState; 719 720 // Last non-search state 721 D6DMachineState lastNonSearchState; 722 }; 723 724 struct D6DState { 725 // The lastest d6d probe packet transmitted 726 folly::Optional<D6DProbePacket> lastProbe; 727 728 // The raise timeout 729 std::chrono::seconds raiseTimeout{kDefaultD6DRaiseTimeout}; 730 731 // The probe timeout 732 std::chrono::seconds probeTimeout{kDefaultD6DProbeTimeout}; 733 734 // The number of outstanding probe packets 735 uint64_t outstandingProbes{0}; 736 737 // The base PMTU to start probing with 738 uint16_t basePMTU{kDefaultD6DBasePMTU}; 739 740 // The max PMTU, determined by max_packet_size transport parameter 741 uint16_t maxPMTU{kDefaultMaxUDPPayload}; 742 743 // Current probe size, dynamically adjusted by the probing algorithm 744 uint32_t currentProbeSize{kDefaultD6DBasePMTU}; 745 746 // Probe size raiser 747 std::unique_ptr<ProbeSizeRaiser> raiser; 748 749 // ThresholdCounter to help detect PMTU blackhole 750 std::unique_ptr<WindowedCounter<uint64_t, uint64_t>> thresholdCounter{ 751 nullptr}; 752 753 // Meta state 754 D6DMetaState meta; 755 756 // Turn off blackhole detection 757 bool noBlackholeDetection{false}; 758 759 // D6D Machine State 760 D6DMachineState state{D6DMachineState::DISABLED}; 761 }; 762 763 D6DState d6d; 764 765 // Debug information. Currently only used to debug busy loop of Transport 766 // WriteLooper. 767 struct WriteDebugState { 768 bool needsWriteLoopDetect{false}; 769 uint64_t currentEmptyLoopCount{0}; 770 WriteDataReason writeDataReason{WriteDataReason::NO_WRITE}; 771 NoWriteReason noWriteReason{NoWriteReason::WRITE_OK}; 772 std::string schedulerName; 773 }; 774 775 struct ReadDebugState { 776 uint64_t loopCount{0}; 777 NoReadReason noReadReason{NoReadReason::READ_OK}; 778 }; 779 780 WriteDebugState writeDebugState; 781 ReadDebugState readDebugState; 782 783 std::shared_ptr<LoopDetectorCallback> loopDetectorCallback; 784 785 // Measure rtt betwen pathchallenge & path response frame 786 // Use this measured rtt as init rtt (from Transport Settings) 787 TimePoint pathChallengeStartTime; 788 789 /** 790 * Eary data app params functions. 791 */ 792 folly::Function<bool(const folly::Optional<std::string>&, const Buf&) const> 793 earlyDataAppParamsValidator; 794 folly::Function<Buf()> earlyDataAppParamsGetter; 795 796 /** 797 * Selects a previously unused peer-issued connection id to use. 798 * If there are no available ids return false and don't change anything. 799 * Return true if replacement succeeds. 800 */ 801 bool retireAndSwitchPeerConnectionIds(); 802 803 // queue of functions to be called in processCallbacksAfterNetworkData 804 std::vector<std::function<void(QuicSocket*)>> pendingCallbacks; 805 806 // Vector of Observers that are attached to this socket. 807 std::shared_ptr<const ObserverVec> observers; 808 809 // Type of node owning this connection (client or server). 810 QuicNodeType nodeType; 811 812 // Whether or not we received a new packet before a write. 813 bool receivedNewPacketBeforeWrite{false}; 814 815 // Whether we've set the transporot parameters from transportSettings yet. 816 bool transportParametersEncoded{false}; 817 818 // Whether a connection can be paced based on its handshake and close states. 819 // For example, we may not want to pace a connection that's still handshaking. 820 bool canBePaced{false}; 821 822 // Flag indicating whether the socket is currently waiting for the app to 823 // write data. All new sockets start off in this state where they wait for the 824 // application to pump data to the socket. 825 bool waitingForAppData{true}; 826 827 // Monotonically increasing counter that is incremented each time there is a 828 // write on this socket (writeSocketData() is called), This is used to 829 // identify specific outstanding packets (based on writeCount and packetNum) 830 // in the Observers, to construct Write Blocks 831 uint64_t writeCount{0}; 832 833 // Number of DSR packets sent by this connection. 834 uint64_t dsrPacketCount{0}; 835 836 // Whether we successfully used 0-RTT keys in this connection. 837 bool usedZeroRtt{false}; 838 839 struct DatagramState { 840 uint16_t maxReadFrameSize{kDefaultMaxDatagramFrameSize}; 841 uint16_t maxWriteFrameSize{kDefaultMaxDatagramFrameSize}; 842 uint32_t maxReadBufferSize{kDefaultMaxDatagramsBuffered}; 843 uint32_t maxWriteBufferSize{kDefaultMaxDatagramsBuffered}; 844 // Buffers Incoming Datagrams 845 std::deque<BufQueue> readBuffer; 846 // Buffers Outgoing Datagrams 847 std::deque<BufQueue> writeBuffer; 848 }; 849 850 DatagramState datagramState; 851 }; 852 853 std::ostream& operator<<(std::ostream& os, const QuicConnectionStateBase& st); 854 855 struct AckStateVersion { 856 uint64_t initialAckStateVersion{kDefaultIntervalSetVersion}; 857 uint64_t handshakeAckStateVersion{kDefaultIntervalSetVersion}; 858 uint64_t appDataAckStateVersion{kDefaultIntervalSetVersion}; 859 860 AckStateVersion( 861 uint64_t initialVersion, 862 uint64_t handshakeVersion, 863 uint64_t appDataVersion); 864 865 bool operator==(const AckStateVersion& other) const; 866 bool operator!=(const AckStateVersion& other) const; 867 }; 868 869 } // namespace quic 870