1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/receiver.h"
12
13 #include <assert.h>
14
15 #include <cstdint>
16 #include <cstdlib>
17 #include <utility>
18 #include <vector>
19
20 #include "absl/memory/memory.h"
21 #include "api/video/encoded_image.h"
22 #include "modules/video_coding/encoded_frame.h"
23 #include "modules/video_coding/internal_defines.h"
24 #include "modules/video_coding/jitter_buffer_common.h"
25 #include "rtc_base/logging.h"
26 #include "rtc_base/numerics/safe_conversions.h"
27 #include "rtc_base/trace_event.h"
28 #include "system_wrappers/include/clock.h"
29
30 namespace webrtc {
31
32 enum { kMaxReceiverDelayMs = 10000 };
33
VCMReceiver(VCMTiming * timing,Clock * clock)34 VCMReceiver::VCMReceiver(VCMTiming* timing, Clock* clock)
35 : VCMReceiver::VCMReceiver(timing,
36 clock,
37 absl::WrapUnique(EventWrapper::Create()),
38 absl::WrapUnique(EventWrapper::Create())) {}
39
VCMReceiver(VCMTiming * timing,Clock * clock,std::unique_ptr<EventWrapper> receiver_event,std::unique_ptr<EventWrapper> jitter_buffer_event)40 VCMReceiver::VCMReceiver(VCMTiming* timing,
41 Clock* clock,
42 std::unique_ptr<EventWrapper> receiver_event,
43 std::unique_ptr<EventWrapper> jitter_buffer_event)
44 : clock_(clock),
45 jitter_buffer_(clock_, std::move(jitter_buffer_event)),
46 timing_(timing),
47 render_wait_event_(std::move(receiver_event)),
48 max_video_delay_ms_(kMaxVideoDelayMs) {
49 jitter_buffer_.Start();
50 }
51
~VCMReceiver()52 VCMReceiver::~VCMReceiver() {
53 render_wait_event_->Set();
54 }
55
InsertPacket(const VCMPacket & packet)56 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) {
57 // Insert the packet into the jitter buffer. The packet can either be empty or
58 // contain media at this point.
59 bool retransmitted = false;
60 const VCMFrameBufferEnum ret =
61 jitter_buffer_.InsertPacket(packet, &retransmitted);
62 if (ret == kOldPacket) {
63 return VCM_OK;
64 } else if (ret == kFlushIndicator) {
65 return VCM_FLUSH_INDICATOR;
66 } else if (ret < 0) {
67 return VCM_JITTER_BUFFER_ERROR;
68 }
69 if (ret == kCompleteSession && !retransmitted) {
70 // We don't want to include timestamps which have suffered from
71 // retransmission here, since we compensate with extra retransmission
72 // delay within the jitter estimate.
73 timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
74 }
75 return VCM_OK;
76 }
77
FrameForDecoding(uint16_t max_wait_time_ms,bool prefer_late_decoding)78 VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
79 bool prefer_late_decoding) {
80 const int64_t start_time_ms = clock_->TimeInMilliseconds();
81 uint32_t frame_timestamp = 0;
82 int min_playout_delay_ms = -1;
83 int max_playout_delay_ms = -1;
84 int64_t render_time_ms = 0;
85 // Exhaust wait time to get a complete frame for decoding.
86 VCMEncodedFrame* found_frame =
87 jitter_buffer_.NextCompleteFrame(max_wait_time_ms);
88
89 if (found_frame) {
90 frame_timestamp = found_frame->Timestamp();
91 min_playout_delay_ms = found_frame->EncodedImage().playout_delay_.min_ms;
92 max_playout_delay_ms = found_frame->EncodedImage().playout_delay_.max_ms;
93 } else {
94 return nullptr;
95 }
96
97 if (min_playout_delay_ms >= 0)
98 timing_->set_min_playout_delay(min_playout_delay_ms);
99
100 if (max_playout_delay_ms >= 0)
101 timing_->set_max_playout_delay(max_playout_delay_ms);
102
103 // We have a frame - Set timing and render timestamp.
104 timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
105 const int64_t now_ms = clock_->TimeInMilliseconds();
106 timing_->UpdateCurrentDelay(frame_timestamp);
107 render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
108 // Check render timing.
109 bool timing_error = false;
110 // Assume that render timing errors are due to changes in the video stream.
111 if (render_time_ms < 0) {
112 timing_error = true;
113 } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) {
114 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
115 RTC_LOG(LS_WARNING)
116 << "A frame about to be decoded is out of the configured "
117 "delay bounds ("
118 << frame_delay << " > " << max_video_delay_ms_
119 << "). Resetting the video jitter buffer.";
120 timing_error = true;
121 } else if (static_cast<int>(timing_->TargetVideoDelay()) >
122 max_video_delay_ms_) {
123 RTC_LOG(LS_WARNING) << "The video target delay has grown larger than "
124 << max_video_delay_ms_
125 << " ms. Resetting jitter buffer.";
126 timing_error = true;
127 }
128
129 if (timing_error) {
130 // Timing error => reset timing and flush the jitter buffer.
131 jitter_buffer_.Flush();
132 timing_->Reset();
133 return NULL;
134 }
135
136 if (prefer_late_decoding) {
137 // Decode frame as close as possible to the render timestamp.
138 const int32_t available_wait_time =
139 max_wait_time_ms -
140 static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
141 uint16_t new_max_wait_time =
142 static_cast<uint16_t>(VCM_MAX(available_wait_time, 0));
143 uint32_t wait_time_ms = rtc::saturated_cast<uint32_t>(
144 timing_->MaxWaitingTime(render_time_ms, clock_->TimeInMilliseconds()));
145 if (new_max_wait_time < wait_time_ms) {
146 // We're not allowed to wait until the frame is supposed to be rendered,
147 // waiting as long as we're allowed to avoid busy looping, and then return
148 // NULL. Next call to this function might return the frame.
149 render_wait_event_->Wait(new_max_wait_time);
150 return NULL;
151 }
152 // Wait until it's time to render.
153 render_wait_event_->Wait(wait_time_ms);
154 }
155
156 // Extract the frame from the jitter buffer and set the render time.
157 VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
158 if (frame == NULL) {
159 return NULL;
160 }
161 frame->SetRenderTime(render_time_ms);
162 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->Timestamp(), "SetRenderTS",
163 "render_time", frame->RenderTimeMs());
164 return frame;
165 }
166
ReleaseFrame(VCMEncodedFrame * frame)167 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
168 jitter_buffer_.ReleaseFrame(frame);
169 }
170
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)171 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
172 int max_packet_age_to_nack,
173 int max_incomplete_time_ms) {
174 jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
175 max_incomplete_time_ms);
176 }
177
NackList(bool * request_key_frame)178 std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) {
179 return jitter_buffer_.GetNackList(request_key_frame);
180 }
181
182 } // namespace webrtc
183