1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_
12 #define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_
13 
14 #include "modules/video_coding/include/video_coding.h"
15 
16 #include <memory>
17 #include <string>
18 #include <vector>
19 
20 #include "common_video/include/frame_callback.h"
21 #include "modules/video_coding/codec_database.h"
22 #include "modules/video_coding/frame_buffer.h"
23 #include "modules/video_coding/generic_decoder.h"
24 #include "modules/video_coding/generic_encoder.h"
25 #include "modules/video_coding/jitter_buffer.h"
26 #include "modules/video_coding/media_optimization.h"
27 #include "modules/video_coding/qp_parser.h"
28 #include "modules/video_coding/receiver.h"
29 #include "modules/video_coding/timing.h"
30 #include "rtc_base/onetimeevent.h"
31 #include "rtc_base/sequenced_task_checker.h"
32 #include "rtc_base/thread_annotations.h"
33 #include "rtc_base/thread_checker.h"
34 #include "system_wrappers/include/clock.h"
35 
36 namespace webrtc {
37 
38 class VideoBitrateAllocator;
39 class VideoBitrateAllocationObserver;
40 
41 namespace vcm {
42 
43 class VCMProcessTimer {
44  public:
45   static const int64_t kDefaultProcessIntervalMs = 1000;
46 
VCMProcessTimer(int64_t periodMs,Clock * clock)47   VCMProcessTimer(int64_t periodMs, Clock* clock)
48       : _clock(clock),
49         _periodMs(periodMs),
50         _latestMs(_clock->TimeInMilliseconds()) {}
51   int64_t Period() const;
52   int64_t TimeUntilProcess() const;
53   void Processed();
54 
55  private:
56   Clock* _clock;
57   int64_t _periodMs;
58   int64_t _latestMs;
59 };
60 
61 class VideoSender {
62  public:
63   typedef VideoCodingModule::SenderNackMode SenderNackMode;
64 
65   VideoSender(Clock* clock,
66               EncodedImageCallback* post_encode_callback);
67 
68   ~VideoSender();
69 
70   // Register the send codec to be used.
71   // This method must be called on the construction thread.
72   int32_t RegisterSendCodec(const VideoCodec* sendCodec,
73                             uint32_t numberOfCores,
74                             uint32_t maxPayloadSize);
75 
76   void RegisterExternalEncoder(VideoEncoder* externalEncoder,
77                                uint8_t payloadType,
78                                bool internalSource);
79 
80   int Bitrate(unsigned int* bitrate) const;
81   int FrameRate(unsigned int* framerate) const;
82 
83   // Update the channel parameters based on new rates and rtt. This will also
84   // cause an immediate call to VideoEncoder::SetRateAllocation().
85   int32_t SetChannelParameters(
86       uint32_t target_bitrate_bps,
87       uint8_t loss_rate,
88       int64_t rtt,
89       VideoBitrateAllocator* bitrate_allocator,
90       VideoBitrateAllocationObserver* bitrate_updated_callback);
91 
92   // Updates the channel parameters with a new bitrate allocation, but using the
93   // current targit_bitrate, loss rate and rtt. That is, the distribution or
94   // caps may be updated to a change to a new VideoCodec or allocation mode.
95   // The new parameters will be stored as pending EncoderParameters, and the
96   // encoder will only be updated on the next frame.
97   void UpdateChannelParemeters(
98       VideoBitrateAllocator* bitrate_allocator,
99       VideoBitrateAllocationObserver* bitrate_updated_callback);
100 
101   // Deprecated:
102   // TODO(perkj): Remove once no projects use it.
103   int32_t RegisterProtectionCallback(VCMProtectionCallback* protection);
104 
105   int32_t AddVideoFrame(const VideoFrame& videoFrame,
106                         const CodecSpecificInfo* codecSpecificInfo);
107 
108   int32_t IntraFrameRequest(size_t stream_index);
109   int32_t EnableFrameDropper(bool enable);
110 
111  private:
112   EncoderParameters UpdateEncoderParameters(
113       const EncoderParameters& params,
114       VideoBitrateAllocator* bitrate_allocator,
115       uint32_t target_bitrate_bps);
116   void SetEncoderParameters(EncoderParameters params, bool has_internal_source)
117       RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_);
118 
119   rtc::CriticalSection encoder_crit_;
120   VCMGenericEncoder* _encoder;
121   media_optimization::MediaOptimization _mediaOpt;
122   VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_);
123   EncodedImageCallback* const post_encode_callback_;
124   VCMCodecDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_);
125   bool frame_dropper_enabled_ RTC_GUARDED_BY(encoder_crit_);
126 
127   // Must be accessed on the construction thread of VideoSender.
128   VideoCodec current_codec_;
129   rtc::SequencedTaskChecker sequenced_checker_;
130 
131   rtc::CriticalSection params_crit_;
132   EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_);
133   bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_);
134   std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_);
135 };
136 
137 class VideoReceiver : public Module {
138  public:
139   VideoReceiver(Clock* clock,
140                 EventFactory* event_factory,
141                 EncodedImageCallback* pre_decode_image_callback,
142                 VCMTiming* timing,
143                 NackSender* nack_sender = nullptr,
144                 KeyFrameRequestSender* keyframe_request_sender = nullptr);
145   ~VideoReceiver();
146 
147   int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
148                                int32_t numberOfCores,
149                                bool requireKeyFrame);
150 
151   void RegisterExternalDecoder(VideoDecoder* externalDecoder,
152                                uint8_t payloadType);
153   int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback);
154   int32_t RegisterReceiveStatisticsCallback(
155       VCMReceiveStatisticsCallback* receiveStats);
156   int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback);
157   int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback);
158 
159   int32_t Decode(uint16_t maxWaitTimeMs);
160 
161   int32_t Decode(const webrtc::VCMEncodedFrame* frame);
162 
163   // Called on the decoder thread when thread is exiting.
164   void DecodingStopped();
165 
166   int32_t IncomingPacket(const uint8_t* incomingPayload,
167                          size_t payloadLength,
168                          const WebRtcRTPHeader& rtpInfo);
169   int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs);
170   int32_t SetRenderDelay(uint32_t timeMS);
171   int32_t Delay() const;
172 
173   // DEPRECATED.
174   int SetReceiverRobustnessMode(
175       VideoCodingModule::ReceiverRobustness robustnessMode,
176       VCMDecodeErrorMode errorMode);
177 
178   void SetNackSettings(size_t max_nack_list_size,
179                        int max_packet_age_to_nack,
180                        int max_incomplete_time_ms);
181 
182   void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode);
183   int SetMinReceiverDelay(int desired_delay_ms);
184 
185   int32_t SetReceiveChannelParameters(int64_t rtt);
186   int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable);
187 
188   int64_t TimeUntilNextProcess() override;
189   void Process() override;
190 
191   void TriggerDecoderShutdown();
192 
193  protected:
194   int32_t Decode(const webrtc::VCMEncodedFrame& frame)
195       RTC_EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
196   int32_t RequestKeyFrame();
197 
198  private:
199   rtc::ThreadChecker construction_thread_;
200   Clock* const clock_;
201   rtc::CriticalSection process_crit_;
202   rtc::CriticalSection receive_crit_;
203   VCMTiming* _timing;
204   VCMReceiver _receiver;
205   VCMDecodedFrameCallback _decodedFrameCallback;
206   VCMFrameTypeCallback* _frameTypeCallback RTC_GUARDED_BY(process_crit_);
207   VCMReceiveStatisticsCallback* _receiveStatsCallback
208       RTC_GUARDED_BY(process_crit_);
209   VCMPacketRequestCallback* _packetRequestCallback
210       RTC_GUARDED_BY(process_crit_);
211 
212   VCMFrameBuffer _frameFromFile;
213   bool _scheduleKeyRequest RTC_GUARDED_BY(process_crit_);
214   bool drop_frames_until_keyframe_ RTC_GUARDED_BY(process_crit_);
215   size_t max_nack_list_size_ RTC_GUARDED_BY(process_crit_);
216 
217   VCMCodecDataBase _codecDataBase RTC_GUARDED_BY(receive_crit_);
218   EncodedImageCallback* pre_decode_image_callback_;
219 
220   VCMProcessTimer _receiveStatsTimer;
221   VCMProcessTimer _retransmissionTimer;
222   VCMProcessTimer _keyRequestTimer;
223   QpParser qp_parser_;
224   ThreadUnsafeOneTimeEvent first_frame_received_;
225 };
226 
227 }  // namespace vcm
228 }  // namespace webrtc
229 #endif  // MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_
230