1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <stddef.h>
12
13 #include <cstdint>
14 #include <vector>
15
16 #include "api/rtp_headers.h"
17 #include "api/video_codecs/video_codec.h"
18 #include "api/video_codecs/video_decoder.h"
19 #include "modules/utility/include/process_thread.h"
20 #include "modules/video_coding/decoder_database.h"
21 #include "modules/video_coding/encoded_frame.h"
22 #include "modules/video_coding/generic_decoder.h"
23 #include "modules/video_coding/include/video_coding.h"
24 #include "modules/video_coding/include/video_coding_defines.h"
25 #include "modules/video_coding/internal_defines.h"
26 #include "modules/video_coding/jitter_buffer.h"
27 #include "modules/video_coding/media_opt_util.h"
28 #include "modules/video_coding/packet.h"
29 #include "modules/video_coding/receiver.h"
30 #include "modules/video_coding/timing.h"
31 #include "modules/video_coding/video_coding_impl.h"
32 #include "rtc_base/checks.h"
33 #include "rtc_base/location.h"
34 #include "rtc_base/logging.h"
35 #include "rtc_base/one_time_event.h"
36 #include "rtc_base/thread_checker.h"
37 #include "rtc_base/trace_event.h"
38 #include "system_wrappers/include/clock.h"
39
40 namespace webrtc {
41 namespace vcm {
42
VideoReceiver(Clock * clock,VCMTiming * timing)43 VideoReceiver::VideoReceiver(Clock* clock, VCMTiming* timing)
44 : clock_(clock),
45 _timing(timing),
46 _receiver(_timing, clock_),
47 _decodedFrameCallback(_timing, clock_),
48 _frameTypeCallback(nullptr),
49 _packetRequestCallback(nullptr),
50 _scheduleKeyRequest(false),
51 drop_frames_until_keyframe_(false),
52 max_nack_list_size_(0),
53 _codecDataBase(),
54 _retransmissionTimer(10, clock_),
55 _keyRequestTimer(500, clock_) {
56 decoder_thread_checker_.Detach();
57 module_thread_checker_.Detach();
58 }
59
~VideoReceiver()60 VideoReceiver::~VideoReceiver() {
61 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
62 }
63
Process()64 void VideoReceiver::Process() {
65 RTC_DCHECK_RUN_ON(&module_thread_checker_);
66
67 // Key frame requests
68 if (_keyRequestTimer.TimeUntilProcess() == 0) {
69 _keyRequestTimer.Processed();
70 bool request_key_frame = _frameTypeCallback != nullptr;
71 if (request_key_frame) {
72 MutexLock lock(&process_mutex_);
73 request_key_frame = _scheduleKeyRequest;
74 }
75 if (request_key_frame)
76 RequestKeyFrame();
77 }
78
79 // Packet retransmission requests
80 // TODO(holmer): Add API for changing Process interval and make sure it's
81 // disabled when NACK is off.
82 if (_retransmissionTimer.TimeUntilProcess() == 0) {
83 _retransmissionTimer.Processed();
84 bool callback_registered = _packetRequestCallback != nullptr;
85 uint16_t length = max_nack_list_size_;
86 if (callback_registered && length > 0) {
87 // Collect sequence numbers from the default receiver.
88 bool request_key_frame = false;
89 std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
90 int32_t ret = VCM_OK;
91 if (request_key_frame) {
92 ret = RequestKeyFrame();
93 }
94 if (ret == VCM_OK && !nackList.empty()) {
95 MutexLock lock(&process_mutex_);
96 if (_packetRequestCallback != nullptr) {
97 _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
98 }
99 }
100 }
101 }
102 }
103
ProcessThreadAttached(ProcessThread * process_thread)104 void VideoReceiver::ProcessThreadAttached(ProcessThread* process_thread) {
105 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
106 if (process_thread) {
107 is_attached_to_process_thread_ = true;
108 RTC_DCHECK(!process_thread_ || process_thread_ == process_thread);
109 process_thread_ = process_thread;
110 } else {
111 is_attached_to_process_thread_ = false;
112 }
113 }
114
TimeUntilNextProcess()115 int64_t VideoReceiver::TimeUntilNextProcess() {
116 RTC_DCHECK_RUN_ON(&module_thread_checker_);
117 int64_t timeUntilNextProcess = _retransmissionTimer.TimeUntilProcess();
118
119 timeUntilNextProcess =
120 VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
121
122 return timeUntilNextProcess;
123 }
124
125 // Register a receive callback. Will be called whenever there is a new frame
126 // ready for rendering.
RegisterReceiveCallback(VCMReceiveCallback * receiveCallback)127 int32_t VideoReceiver::RegisterReceiveCallback(
128 VCMReceiveCallback* receiveCallback) {
129 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
130 // This value is set before the decoder thread starts and unset after
131 // the decoder thread has been stopped.
132 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
133 return VCM_OK;
134 }
135
136 // Register an externally defined decoder object.
RegisterExternalDecoder(VideoDecoder * externalDecoder,uint8_t payloadType)137 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
138 uint8_t payloadType) {
139 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
140 if (externalDecoder == nullptr) {
141 RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
142 return;
143 }
144 _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
145 }
146
147 // Register a frame type request callback.
RegisterFrameTypeCallback(VCMFrameTypeCallback * frameTypeCallback)148 int32_t VideoReceiver::RegisterFrameTypeCallback(
149 VCMFrameTypeCallback* frameTypeCallback) {
150 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
151 RTC_DCHECK(!is_attached_to_process_thread_);
152 // This callback is used on the module thread, but since we don't get
153 // callbacks on the module thread while the decoder thread isn't running
154 // (and this function must not be called when the decoder is running),
155 // we don't need a lock here.
156 _frameTypeCallback = frameTypeCallback;
157 return VCM_OK;
158 }
159
RegisterPacketRequestCallback(VCMPacketRequestCallback * callback)160 int32_t VideoReceiver::RegisterPacketRequestCallback(
161 VCMPacketRequestCallback* callback) {
162 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
163 RTC_DCHECK(!is_attached_to_process_thread_);
164 // This callback is used on the module thread, but since we don't get
165 // callbacks on the module thread while the decoder thread isn't running
166 // (and this function must not be called when the decoder is running),
167 // we don't need a lock here.
168 _packetRequestCallback = callback;
169 return VCM_OK;
170 }
171
172 // Decode next frame, blocking.
173 // Should be called as often as possible to get the most out of the decoder.
Decode(uint16_t maxWaitTimeMs)174 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
175 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
176 VCMEncodedFrame* frame = _receiver.FrameForDecoding(
177 maxWaitTimeMs, _codecDataBase.PrefersLateDecoding());
178
179 if (!frame)
180 return VCM_FRAME_NOT_READY;
181
182 bool drop_frame = false;
183 {
184 MutexLock lock(&process_mutex_);
185 if (drop_frames_until_keyframe_) {
186 // Still getting delta frames, schedule another keyframe request as if
187 // decode failed.
188 if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
189 drop_frame = true;
190 _scheduleKeyRequest = true;
191 // TODO(tommi): Consider if we could instead post a task to the module
192 // thread and call RequestKeyFrame directly. Here we call WakeUp so that
193 // TimeUntilNextProcess() gets called straight away.
194 process_thread_->WakeUp(this);
195 } else {
196 drop_frames_until_keyframe_ = false;
197 }
198 }
199 }
200
201 if (drop_frame) {
202 _receiver.ReleaseFrame(frame);
203 return VCM_FRAME_NOT_READY;
204 }
205
206 // If this frame was too late, we should adjust the delay accordingly
207 _timing->UpdateCurrentDelay(frame->RenderTimeMs(),
208 clock_->TimeInMilliseconds());
209
210 if (first_frame_received_()) {
211 RTC_LOG(LS_INFO) << "Received first "
212 << (frame->Complete() ? "complete" : "incomplete")
213 << " decodable video frame";
214 }
215
216 const int32_t ret = Decode(*frame);
217 _receiver.ReleaseFrame(frame);
218 return ret;
219 }
220
RequestKeyFrame()221 int32_t VideoReceiver::RequestKeyFrame() {
222 RTC_DCHECK_RUN_ON(&module_thread_checker_);
223
224 TRACE_EVENT0("webrtc", "RequestKeyFrame");
225 if (_frameTypeCallback != nullptr) {
226 const int32_t ret = _frameTypeCallback->RequestKeyFrame();
227 if (ret < 0) {
228 return ret;
229 }
230 MutexLock lock(&process_mutex_);
231 _scheduleKeyRequest = false;
232 } else {
233 return VCM_MISSING_CALLBACK;
234 }
235 return VCM_OK;
236 }
237
238 // Must be called from inside the receive side critical section.
Decode(const VCMEncodedFrame & frame)239 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
240 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
241 TRACE_EVENT0("webrtc", "VideoReceiver::Decode");
242 // Change decoder if payload type has changed
243 VCMGenericDecoder* decoder =
244 _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
245 if (decoder == nullptr) {
246 return VCM_NO_CODEC_REGISTERED;
247 }
248 return decoder->Decode(frame, clock_->CurrentTime());
249 }
250
251 // Register possible receive codecs, can be called multiple times
RegisterReceiveCodec(uint8_t payload_type,const VideoCodec * receiveCodec,int32_t numberOfCores)252 int32_t VideoReceiver::RegisterReceiveCodec(uint8_t payload_type,
253 const VideoCodec* receiveCodec,
254 int32_t numberOfCores) {
255 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
256 if (receiveCodec == nullptr) {
257 return VCM_PARAMETER_ERROR;
258 }
259 if (!_codecDataBase.RegisterReceiveCodec(payload_type, receiveCodec,
260 numberOfCores)) {
261 return -1;
262 }
263 return 0;
264 }
265
266 // Incoming packet from network parsed and ready for decode, non blocking.
IncomingPacket(const uint8_t * incomingPayload,size_t payloadLength,const RTPHeader & rtp_header,const RTPVideoHeader & video_header)267 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
268 size_t payloadLength,
269 const RTPHeader& rtp_header,
270 const RTPVideoHeader& video_header) {
271 RTC_DCHECK_RUN_ON(&module_thread_checker_);
272 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
273 TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
274 rtp_header.sequenceNumber);
275 }
276 if (incomingPayload == nullptr) {
277 // The jitter buffer doesn't handle non-zero payload lengths for packets
278 // without payload.
279 // TODO(holmer): We should fix this in the jitter buffer.
280 payloadLength = 0;
281 }
282 // Callers don't provide any ntp time.
283 const VCMPacket packet(incomingPayload, payloadLength, rtp_header,
284 video_header, /*ntp_time_ms=*/0,
285 clock_->TimeInMilliseconds());
286 int32_t ret = _receiver.InsertPacket(packet);
287
288 // TODO(holmer): Investigate if this somehow should use the key frame
289 // request scheduling to throttle the requests.
290 if (ret == VCM_FLUSH_INDICATOR) {
291 {
292 MutexLock lock(&process_mutex_);
293 drop_frames_until_keyframe_ = true;
294 }
295 RequestKeyFrame();
296 } else if (ret < 0) {
297 return ret;
298 }
299 return VCM_OK;
300 }
301
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)302 void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
303 int max_packet_age_to_nack,
304 int max_incomplete_time_ms) {
305 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
306 if (max_nack_list_size != 0) {
307 max_nack_list_size_ = max_nack_list_size;
308 }
309 _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
310 max_incomplete_time_ms);
311 }
312
313 } // namespace vcm
314 } // namespace webrtc
315