1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ 12 #define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ 13 14 #include "modules/video_coding/include/video_coding.h" 15 16 #include <memory> 17 #include <string> 18 #include <vector> 19 20 #include "common_video/include/frame_callback.h" 21 #include "modules/video_coding/codec_database.h" 22 #include "modules/video_coding/frame_buffer.h" 23 #include "modules/video_coding/generic_decoder.h" 24 #include "modules/video_coding/generic_encoder.h" 25 #include "modules/video_coding/jitter_buffer.h" 26 #include "modules/video_coding/media_optimization.h" 27 #include "modules/video_coding/qp_parser.h" 28 #include "modules/video_coding/receiver.h" 29 #include "modules/video_coding/timing.h" 30 #include "rtc_base/onetimeevent.h" 31 #include "rtc_base/sequenced_task_checker.h" 32 #include "rtc_base/thread_annotations.h" 33 #include "rtc_base/thread_checker.h" 34 #include "system_wrappers/include/clock.h" 35 36 namespace webrtc { 37 38 class VideoBitrateAllocator; 39 class VideoBitrateAllocationObserver; 40 41 namespace vcm { 42 43 class VCMProcessTimer { 44 public: 45 static const int64_t kDefaultProcessIntervalMs = 1000; 46 VCMProcessTimer(int64_t periodMs,Clock * clock)47 VCMProcessTimer(int64_t periodMs, Clock* clock) 48 : _clock(clock), 49 _periodMs(periodMs), 50 _latestMs(_clock->TimeInMilliseconds()) {} 51 int64_t Period() const; 52 int64_t TimeUntilProcess() const; 53 void Processed(); 54 55 private: 56 Clock* _clock; 57 int64_t _periodMs; 58 int64_t _latestMs; 59 }; 60 61 class VideoSender { 62 public: 63 typedef VideoCodingModule::SenderNackMode SenderNackMode; 64 65 VideoSender(Clock* clock, 66 EncodedImageCallback* post_encode_callback); 67 68 ~VideoSender(); 69 70 // Register the send codec to be used. 71 // This method must be called on the construction thread. 72 int32_t RegisterSendCodec(const VideoCodec* sendCodec, 73 uint32_t numberOfCores, 74 uint32_t maxPayloadSize); 75 76 void RegisterExternalEncoder(VideoEncoder* externalEncoder, 77 uint8_t payloadType, 78 bool internalSource); 79 80 int Bitrate(unsigned int* bitrate) const; 81 int FrameRate(unsigned int* framerate) const; 82 83 // Update the channel parameters based on new rates and rtt. This will also 84 // cause an immediate call to VideoEncoder::SetRateAllocation(). 85 int32_t SetChannelParameters( 86 uint32_t target_bitrate_bps, 87 uint8_t loss_rate, 88 int64_t rtt, 89 VideoBitrateAllocator* bitrate_allocator, 90 VideoBitrateAllocationObserver* bitrate_updated_callback); 91 92 // Updates the channel parameters with a new bitrate allocation, but using the 93 // current targit_bitrate, loss rate and rtt. That is, the distribution or 94 // caps may be updated to a change to a new VideoCodec or allocation mode. 95 // The new parameters will be stored as pending EncoderParameters, and the 96 // encoder will only be updated on the next frame. 97 void UpdateChannelParemeters( 98 VideoBitrateAllocator* bitrate_allocator, 99 VideoBitrateAllocationObserver* bitrate_updated_callback); 100 101 // Deprecated: 102 // TODO(perkj): Remove once no projects use it. 103 int32_t RegisterProtectionCallback(VCMProtectionCallback* protection); 104 105 int32_t AddVideoFrame(const VideoFrame& videoFrame, 106 const CodecSpecificInfo* codecSpecificInfo); 107 108 int32_t IntraFrameRequest(size_t stream_index); 109 int32_t EnableFrameDropper(bool enable); 110 111 private: 112 EncoderParameters UpdateEncoderParameters( 113 const EncoderParameters& params, 114 VideoBitrateAllocator* bitrate_allocator, 115 uint32_t target_bitrate_bps); 116 void SetEncoderParameters(EncoderParameters params, bool has_internal_source) 117 RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_); 118 119 rtc::CriticalSection encoder_crit_; 120 VCMGenericEncoder* _encoder; 121 media_optimization::MediaOptimization _mediaOpt; 122 VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_); 123 EncodedImageCallback* const post_encode_callback_; 124 VCMCodecDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_); 125 bool frame_dropper_enabled_ RTC_GUARDED_BY(encoder_crit_); 126 127 // Must be accessed on the construction thread of VideoSender. 128 VideoCodec current_codec_; 129 rtc::SequencedTaskChecker sequenced_checker_; 130 131 rtc::CriticalSection params_crit_; 132 EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_); 133 bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_); 134 std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_); 135 }; 136 137 class VideoReceiver : public Module { 138 public: 139 VideoReceiver(Clock* clock, 140 EventFactory* event_factory, 141 EncodedImageCallback* pre_decode_image_callback, 142 VCMTiming* timing, 143 NackSender* nack_sender = nullptr, 144 KeyFrameRequestSender* keyframe_request_sender = nullptr); 145 ~VideoReceiver(); 146 147 int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, 148 int32_t numberOfCores, 149 bool requireKeyFrame); 150 151 void RegisterExternalDecoder(VideoDecoder* externalDecoder, 152 uint8_t payloadType); 153 int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); 154 int32_t RegisterReceiveStatisticsCallback( 155 VCMReceiveStatisticsCallback* receiveStats); 156 int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); 157 int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); 158 159 int32_t Decode(uint16_t maxWaitTimeMs); 160 161 int32_t Decode(const webrtc::VCMEncodedFrame* frame); 162 163 // Called on the decoder thread when thread is exiting. 164 void DecodingStopped(); 165 166 int32_t IncomingPacket(const uint8_t* incomingPayload, 167 size_t payloadLength, 168 const WebRtcRTPHeader& rtpInfo); 169 int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs); 170 int32_t SetRenderDelay(uint32_t timeMS); 171 int32_t Delay() const; 172 173 // DEPRECATED. 174 int SetReceiverRobustnessMode( 175 VideoCodingModule::ReceiverRobustness robustnessMode, 176 VCMDecodeErrorMode errorMode); 177 178 void SetNackSettings(size_t max_nack_list_size, 179 int max_packet_age_to_nack, 180 int max_incomplete_time_ms); 181 182 void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode); 183 int SetMinReceiverDelay(int desired_delay_ms); 184 185 int32_t SetReceiveChannelParameters(int64_t rtt); 186 int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable); 187 188 int64_t TimeUntilNextProcess() override; 189 void Process() override; 190 191 void TriggerDecoderShutdown(); 192 193 protected: 194 int32_t Decode(const webrtc::VCMEncodedFrame& frame) 195 RTC_EXCLUSIVE_LOCKS_REQUIRED(receive_crit_); 196 int32_t RequestKeyFrame(); 197 198 private: 199 rtc::ThreadChecker construction_thread_; 200 Clock* const clock_; 201 rtc::CriticalSection process_crit_; 202 rtc::CriticalSection receive_crit_; 203 VCMTiming* _timing; 204 VCMReceiver _receiver; 205 VCMDecodedFrameCallback _decodedFrameCallback; 206 VCMFrameTypeCallback* _frameTypeCallback RTC_GUARDED_BY(process_crit_); 207 VCMReceiveStatisticsCallback* _receiveStatsCallback 208 RTC_GUARDED_BY(process_crit_); 209 VCMPacketRequestCallback* _packetRequestCallback 210 RTC_GUARDED_BY(process_crit_); 211 212 VCMFrameBuffer _frameFromFile; 213 bool _scheduleKeyRequest RTC_GUARDED_BY(process_crit_); 214 bool drop_frames_until_keyframe_ RTC_GUARDED_BY(process_crit_); 215 size_t max_nack_list_size_ RTC_GUARDED_BY(process_crit_); 216 217 VCMCodecDataBase _codecDataBase RTC_GUARDED_BY(receive_crit_); 218 EncodedImageCallback* pre_decode_image_callback_; 219 220 VCMProcessTimer _receiveStatsTimer; 221 VCMProcessTimer _retransmissionTimer; 222 VCMProcessTimer _keyRequestTimer; 223 QpParser qp_parser_; 224 ThreadUnsafeOneTimeEvent first_frame_received_; 225 }; 226 227 } // namespace vcm 228 } // namespace webrtc 229 #endif // MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ 230