1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/video_coding/receiver.h"
12
13 #include <assert.h>
14
15 #include <cstdlib>
16 #include <utility>
17 #include <vector>
18
19 #include "webrtc/base/logging.h"
20 #include "webrtc/base/trace_event.h"
21 #include "webrtc/modules/video_coding/encoded_frame.h"
22 #include "webrtc/modules/video_coding/internal_defines.h"
23 #include "webrtc/modules/video_coding/media_opt_util.h"
24 #include "webrtc/system_wrappers/include/clock.h"
25
26 namespace webrtc {
27
28 enum { kMaxReceiverDelayMs = 10000 };
29
VCMReceiver(VCMTiming * timing,Clock * clock,EventFactory * event_factory)30 VCMReceiver::VCMReceiver(VCMTiming* timing,
31 Clock* clock,
32 EventFactory* event_factory)
33 : VCMReceiver::VCMReceiver(timing,
34 clock,
35 event_factory,
36 nullptr, // NackSender
37 nullptr) // KeyframeRequestSender
38 {}
39
VCMReceiver(VCMTiming * timing,Clock * clock,EventFactory * event_factory,NackSender * nack_sender,KeyFrameRequestSender * keyframe_request_sender)40 VCMReceiver::VCMReceiver(VCMTiming* timing,
41 Clock* clock,
42 EventFactory* event_factory,
43 NackSender* nack_sender,
44 KeyFrameRequestSender* keyframe_request_sender)
45 : VCMReceiver(
46 timing,
47 clock,
48 std::unique_ptr<EventWrapper>(event_factory
49 ? event_factory->CreateEvent()
50 : EventWrapper::Create()),
51 std::unique_ptr<EventWrapper>(event_factory
52 ? event_factory->CreateEvent()
53 : EventWrapper::Create()),
54 nack_sender,
55 keyframe_request_sender) {}
56
VCMReceiver(VCMTiming * timing,Clock * clock,std::unique_ptr<EventWrapper> receiver_event,std::unique_ptr<EventWrapper> jitter_buffer_event)57 VCMReceiver::VCMReceiver(VCMTiming* timing,
58 Clock* clock,
59 std::unique_ptr<EventWrapper> receiver_event,
60 std::unique_ptr<EventWrapper> jitter_buffer_event)
61 : VCMReceiver::VCMReceiver(timing,
62 clock,
63 std::move(receiver_event),
64 std::move(jitter_buffer_event),
65 nullptr, // NackSender
66 nullptr) // KeyframeRequestSender
67 {}
68
VCMReceiver(VCMTiming * timing,Clock * clock,std::unique_ptr<EventWrapper> receiver_event,std::unique_ptr<EventWrapper> jitter_buffer_event,NackSender * nack_sender,KeyFrameRequestSender * keyframe_request_sender)69 VCMReceiver::VCMReceiver(VCMTiming* timing,
70 Clock* clock,
71 std::unique_ptr<EventWrapper> receiver_event,
72 std::unique_ptr<EventWrapper> jitter_buffer_event,
73 NackSender* nack_sender,
74 KeyFrameRequestSender* keyframe_request_sender)
75 : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
76 clock_(clock),
77 jitter_buffer_(clock_,
78 std::move(jitter_buffer_event),
79 nack_sender,
80 keyframe_request_sender),
81 timing_(timing),
82 render_wait_event_(std::move(receiver_event)),
83 receiveState_(kReceiveStateInitial),
84 max_video_delay_ms_(kMaxVideoDelayMs) {
85 Reset();
86 }
87
~VCMReceiver()88 VCMReceiver::~VCMReceiver() {
89 render_wait_event_->Set();
90 delete crit_sect_;
91 }
92
Reset()93 void VCMReceiver::Reset() {
94 CriticalSectionScoped cs(crit_sect_);
95 if (!jitter_buffer_.Running()) {
96 jitter_buffer_.Start();
97 } else {
98 jitter_buffer_.Flush();
99 }
100 receiveState_ = kReceiveStateInitial;
101 }
102
UpdateRtt(int64_t rtt)103 void VCMReceiver::UpdateRtt(int64_t rtt) {
104 jitter_buffer_.UpdateRtt(rtt);
105 }
106
InsertPacket(const VCMPacket & packet)107 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) {
108 // Insert the packet into the jitter buffer. The packet can either be empty or
109 // contain media at this point.
110 bool retransmitted = false;
111 const VCMFrameBufferEnum ret =
112 jitter_buffer_.InsertPacket(packet, &retransmitted);
113 if (ret == kOldPacket) {
114 return VCM_OK;
115 } else if (ret == kFlushIndicator) {
116 return VCM_FLUSH_INDICATOR;
117 } else if (ret < 0) {
118 return VCM_JITTER_BUFFER_ERROR;
119 }
120 if (ret == kCompleteSession && !retransmitted) {
121 // We don't want to include timestamps which have suffered from
122 // retransmission here, since we compensate with extra retransmission
123 // delay within the jitter estimate.
124 timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
125 }
126 return VCM_OK;
127 }
128
TriggerDecoderShutdown()129 void VCMReceiver::TriggerDecoderShutdown() {
130 jitter_buffer_.Stop();
131 render_wait_event_->Set();
132 }
133
FrameForDecoding(uint16_t max_wait_time_ms,bool prefer_late_decoding)134 VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
135 bool prefer_late_decoding) {
136 const int64_t start_time_ms = clock_->TimeInMilliseconds();
137 uint32_t frame_timestamp = 0;
138 int min_playout_delay_ms = -1;
139 int max_playout_delay_ms = -1;
140 int64_t render_time_ms = 0;
141 // Exhaust wait time to get a complete frame for decoding.
142 VCMEncodedFrame* found_frame =
143 jitter_buffer_.NextCompleteFrame(max_wait_time_ms);
144
145 if (found_frame) {
146 frame_timestamp = found_frame->TimeStamp();
147 min_playout_delay_ms = found_frame->EncodedImage().playout_delay_.min_ms;
148 max_playout_delay_ms = found_frame->EncodedImage().playout_delay_.max_ms;
149 } else {
150 if (!jitter_buffer_.NextMaybeIncompleteTimestamp(&frame_timestamp))
151 return nullptr;
152 }
153
154 if (min_playout_delay_ms >= 0)
155 timing_->set_min_playout_delay(min_playout_delay_ms);
156
157 if (max_playout_delay_ms >= 0)
158 timing_->set_max_playout_delay(max_playout_delay_ms);
159
160 // We have a frame - Set timing and render timestamp.
161 timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
162 const int64_t now_ms = clock_->TimeInMilliseconds();
163 timing_->UpdateCurrentDelay(frame_timestamp);
164 render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
165 // Check render timing.
166 bool timing_error = false;
167 // Assume that render timing errors are due to changes in the video stream.
168 if (render_time_ms < 0) {
169 timing_error = true;
170 } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) {
171 int frame_delay = std::abs(render_time_ms - now_ms);
172 LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
173 << "delay bounds (" << frame_delay << " > "
174 << max_video_delay_ms_
175 << "). Resetting the video jitter buffer.";
176 timing_error = true;
177 } else if (static_cast<int>(timing_->TargetVideoDelay()) >
178 max_video_delay_ms_) {
179 LOG(LS_WARNING) << "The video target delay has grown larger than "
180 << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
181 timing_error = true;
182 }
183
184 if (timing_error) {
185 // Timing error => reset timing and flush the jitter buffer.
186 jitter_buffer_.Flush();
187 timing_->Reset();
188 return NULL;
189 }
190
191 if (prefer_late_decoding) {
192 // Decode frame as close as possible to the render timestamp.
193 const int32_t available_wait_time =
194 max_wait_time_ms -
195 static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
196 uint16_t new_max_wait_time =
197 static_cast<uint16_t>(VCM_MAX(available_wait_time, 0));
198 uint32_t wait_time_ms =
199 timing_->MaxWaitingTime(render_time_ms, clock_->TimeInMilliseconds());
200 if (new_max_wait_time < wait_time_ms) {
201 // We're not allowed to wait until the frame is supposed to be rendered,
202 // waiting as long as we're allowed to avoid busy looping, and then return
203 // NULL. Next call to this function might return the frame.
204 render_wait_event_->Wait(new_max_wait_time);
205 return NULL;
206 }
207 // Wait until it's time to render.
208 render_wait_event_->Wait(wait_time_ms);
209 }
210
211 // Extract the frame from the jitter buffer and set the render time.
212 VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
213 if (frame == NULL) {
214 return NULL;
215 }
216 frame->SetRenderTime(render_time_ms);
217 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(), "SetRenderTS",
218 "render_time", frame->RenderTimeMs());
219 UpdateReceiveState(*frame);
220 if (!frame->Complete()) {
221 // Update stats for incomplete frames.
222 bool retransmitted = false;
223 const int64_t last_packet_time_ms =
224 jitter_buffer_.LastPacketTime(frame, &retransmitted);
225 if (last_packet_time_ms >= 0 && !retransmitted) {
226 // We don't want to include timestamps which have suffered from
227 // retransmission here, since we compensate with extra retransmission
228 // delay within the jitter estimate.
229 timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
230 }
231 }
232 return frame;
233 }
234
ReleaseFrame(VCMEncodedFrame * frame)235 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
236 jitter_buffer_.ReleaseFrame(frame);
237 }
238
ReceiveStatistics(uint32_t * bitrate,uint32_t * framerate)239 void VCMReceiver::ReceiveStatistics(uint32_t* bitrate, uint32_t* framerate) {
240 assert(bitrate);
241 assert(framerate);
242 jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
243 }
244
DiscardedPackets() const245 uint32_t VCMReceiver::DiscardedPackets() const {
246 return jitter_buffer_.num_discarded_packets();
247 }
248
SetNackMode(VCMNackMode nackMode,int64_t low_rtt_nack_threshold_ms,int64_t high_rtt_nack_threshold_ms)249 void VCMReceiver::SetNackMode(VCMNackMode nackMode,
250 int64_t low_rtt_nack_threshold_ms,
251 int64_t high_rtt_nack_threshold_ms) {
252 CriticalSectionScoped cs(crit_sect_);
253 // Default to always having NACK enabled in hybrid mode.
254 jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
255 high_rtt_nack_threshold_ms);
256 }
257
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)258 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
259 int max_packet_age_to_nack,
260 int max_incomplete_time_ms) {
261 jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
262 max_incomplete_time_ms);
263 }
264
NackMode() const265 VCMNackMode VCMReceiver::NackMode() const {
266 CriticalSectionScoped cs(crit_sect_);
267 return jitter_buffer_.nack_mode();
268 }
269
NackList(bool * request_key_frame)270 std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) {
271 return jitter_buffer_.GetNackList(request_key_frame);
272 }
273
ReceiveState() const274 VideoReceiveState VCMReceiver::ReceiveState() const {
275 CriticalSectionScoped cs(crit_sect_);
276 return receiveState_;
277 }
278
SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode)279 void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
280 jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
281 }
282
DecodeErrorMode() const283 VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
284 return jitter_buffer_.decode_error_mode();
285 }
286
SetMinReceiverDelay(int desired_delay_ms)287 int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
288 CriticalSectionScoped cs(crit_sect_);
289 if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
290 return -1;
291 }
292 max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
293 // Initializing timing to the desired delay.
294 timing_->set_min_playout_delay(desired_delay_ms);
295 return 0;
296 }
297
UpdateReceiveState(const VCMEncodedFrame & frame)298 void VCMReceiver::UpdateReceiveState(const VCMEncodedFrame& frame) {
299 if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
300 receiveState_ = kReceiveStateNormal;
301 return;
302 }
303 if (frame.MissingFrame() || !frame.Complete()) {
304 // State is corrupted
305 receiveState_ = kReceiveStateWaitingKey;
306 }
307 // state continues
308 }
309
RegisterStatsCallback(VCMReceiveStatisticsCallback * callback)310 void VCMReceiver::RegisterStatsCallback(
311 VCMReceiveStatisticsCallback* callback) {
312 jitter_buffer_.RegisterStatsCallback(callback);
313 }
314
315 } // namespace webrtc
316