1 /* 2 * Copyright 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 11 #define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 12 13 #include <stddef.h> 14 #include <stdint.h> 15 16 #include <atomic> 17 #include <map> 18 #include <memory> 19 #include <vector> 20 21 #include "absl/types/optional.h" 22 #include "api/fec_controller.h" 23 #include "api/rtc_event_log/rtc_event_log.h" 24 #include "api/video/encoded_image.h" 25 #include "api/video/video_bitrate_allocation.h" 26 #include "api/video/video_bitrate_allocator.h" 27 #include "api/video/video_stream_encoder_interface.h" 28 #include "api/video_codecs/video_encoder.h" 29 #include "api/video_codecs/video_encoder_config.h" 30 #include "call/bitrate_allocator.h" 31 #include "call/rtp_config.h" 32 #include "call/rtp_transport_controller_send_interface.h" 33 #include "call/rtp_video_sender_interface.h" 34 #include "modules/include/module_common_types.h" 35 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 36 #include "modules/utility/include/process_thread.h" 37 #include "modules/video_coding/include/video_codec_interface.h" 38 #include "rtc_base/experiments/field_trial_parser.h" 39 #include "rtc_base/synchronization/mutex.h" 40 #include "rtc_base/task_queue.h" 41 #include "rtc_base/task_utils/repeating_task.h" 42 #include "rtc_base/thread_annotations.h" 43 #include "rtc_base/weak_ptr.h" 44 #include "video/encoder_rtcp_feedback.h" 45 #include "video/send_delay_stats.h" 46 #include "video/send_statistics_proxy.h" 47 #include "video/video_send_stream.h" 48 49 namespace webrtc { 50 namespace internal { 51 52 // Pacing buffer config; overridden by ALR config if provided. 53 struct PacingConfig { 54 PacingConfig(); 55 PacingConfig(const PacingConfig&); 56 PacingConfig& operator=(const PacingConfig&) = default; 57 ~PacingConfig(); 58 FieldTrialParameter<double> pacing_factor; 59 FieldTrialParameter<TimeDelta> max_pacing_delay; 60 }; 61 62 // VideoSendStreamImpl implements internal::VideoSendStream. 63 // It is created and destroyed on |worker_queue|. The intent is to decrease the 64 // need for locking and to ensure methods are called in sequence. 65 // Public methods except |DeliverRtcp| must be called on |worker_queue|. 66 // DeliverRtcp is called on the libjingle worker thread or a network thread. 67 // An encoder may deliver frames through the EncodedImageCallback on an 68 // arbitrary thread. 69 class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, 70 public VideoStreamEncoderInterface::EncoderSink, 71 public VideoBitrateAllocationObserver { 72 public: 73 VideoSendStreamImpl( 74 Clock* clock, 75 SendStatisticsProxy* stats_proxy, 76 rtc::TaskQueue* worker_queue, 77 RtcpRttStats* call_stats, 78 RtpTransportControllerSendInterface* transport, 79 BitrateAllocatorInterface* bitrate_allocator, 80 SendDelayStats* send_delay_stats, 81 VideoStreamEncoderInterface* video_stream_encoder, 82 RtcEventLog* event_log, 83 const VideoSendStream::Config* config, 84 int initial_encoder_max_bitrate, 85 double initial_encoder_bitrate_priority, 86 std::map<uint32_t, RtpState> suspended_ssrcs, 87 std::map<uint32_t, RtpPayloadState> suspended_payload_states, 88 VideoEncoderConfig::ContentType content_type, 89 std::unique_ptr<FecController> fec_controller); 90 ~VideoSendStreamImpl() override; 91 92 // RegisterProcessThread register |module_process_thread| with those objects 93 // that use it. Registration has to happen on the thread were 94 // |module_process_thread| was created (libjingle's worker thread). 95 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, 96 // maybe |worker_queue|. 97 void RegisterProcessThread(ProcessThread* module_process_thread); 98 void DeRegisterProcessThread(); 99 100 void DeliverRtcp(const uint8_t* packet, size_t length); 101 void UpdateActiveSimulcastLayers(const std::vector<bool> active_layers); 102 void Start(); 103 void Stop(); 104 105 // TODO(holmer): Move these to RtpTransportControllerSend. 106 std::map<uint32_t, RtpState> GetRtpStates() const; 107 108 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const; 109 110 absl::optional<float> configured_pacing_factor_; 111 112 private: 113 // Implements BitrateAllocatorObserver. 114 uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; 115 116 void OnEncoderConfigurationChanged( 117 std::vector<VideoStream> streams, 118 bool is_svc, 119 VideoEncoderConfig::ContentType content_type, 120 int min_transmit_bitrate_bps) override; 121 122 // Implements EncodedImageCallback. The implementation routes encoded frames 123 // to the |payload_router_| and |config.pre_encode_callback| if set. 124 // Called on an arbitrary encoder callback thread. 125 EncodedImageCallback::Result OnEncodedImage( 126 const EncodedImage& encoded_image, 127 const CodecSpecificInfo* codec_specific_info) override; 128 129 // Implements EncodedImageCallback. 130 void OnDroppedFrame(EncodedImageCallback::DropReason reason) override; 131 132 // Implements VideoBitrateAllocationObserver. 133 void OnBitrateAllocationUpdated( 134 const VideoBitrateAllocation& allocation) override; 135 136 // Starts monitoring and sends a keyframe. 137 void StartupVideoSendStream(); 138 // Removes the bitrate observer, stops monitoring and notifies the video 139 // encoder of the bitrate update. 140 void StopVideoSendStream() RTC_RUN_ON(worker_queue_); 141 142 void ConfigureProtection(); 143 void ConfigureSsrcs(); 144 void SignalEncoderTimedOut(); 145 void SignalEncoderActive(); 146 MediaStreamAllocationConfig GetAllocationConfig() const 147 RTC_RUN_ON(worker_queue_); 148 Clock* const clock_; 149 const bool has_alr_probing_; 150 const PacingConfig pacing_config_; 151 152 SendStatisticsProxy* const stats_proxy_; 153 const VideoSendStream::Config* const config_; 154 155 rtc::TaskQueue* const worker_queue_; 156 157 RepeatingTaskHandle check_encoder_activity_task_ 158 RTC_GUARDED_BY(worker_queue_); 159 160 std::atomic_bool activity_; 161 bool timed_out_ RTC_GUARDED_BY(worker_queue_); 162 163 RtpTransportControllerSendInterface* const transport_; 164 BitrateAllocatorInterface* const bitrate_allocator_; 165 166 Mutex ivf_writers_mutex_; 167 168 bool disable_padding_; 169 int max_padding_bitrate_; 170 int encoder_min_bitrate_bps_; 171 uint32_t encoder_max_bitrate_bps_; 172 uint32_t encoder_target_rate_bps_; 173 double encoder_bitrate_priority_; 174 bool has_packet_feedback_; 175 176 VideoStreamEncoderInterface* const video_stream_encoder_; 177 EncoderRtcpFeedback encoder_feedback_; 178 179 RtcpBandwidthObserver* const bandwidth_observer_; 180 RtpVideoSenderInterface* const rtp_video_sender_; 181 182 // |weak_ptr_| to our self. This is used since we can not call 183 // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy 184 // an existing WeakPtr. 185 rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_; 186 // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are 187 // invalidated before any other members are destroyed. 188 rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_; 189 190 // Context for the most recent and last sent video bitrate allocation. Used to 191 // throttle sending of similar bitrate allocations. 192 struct VbaSendContext { 193 VideoBitrateAllocation last_sent_allocation; 194 absl::optional<VideoBitrateAllocation> throttled_allocation; 195 int64_t last_send_time_ms; 196 }; 197 absl::optional<VbaSendContext> video_bitrate_allocation_context_ 198 RTC_GUARDED_BY(worker_queue_); 199 }; 200 } // namespace internal 201 } // namespace webrtc 202 #endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 203