1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/stream_synchronization.h"
12
13 #include <stdlib.h>
14
15 #include <algorithm>
16
17 #include "rtc_base/logging.h"
18
19 namespace webrtc {
20
21 static const int kMaxChangeMs = 80;
22 static const int kMaxDeltaDelayMs = 10000;
23 static const int kFilterLength = 4;
24 // Minimum difference between audio and video to warrant a change.
25 static const int kMinDeltaMs = 30;
26
StreamSynchronization(uint32_t video_stream_id,uint32_t audio_stream_id)27 StreamSynchronization::StreamSynchronization(uint32_t video_stream_id,
28 uint32_t audio_stream_id)
29 : video_stream_id_(video_stream_id),
30 audio_stream_id_(audio_stream_id),
31 base_target_delay_ms_(0),
32 avg_diff_ms_(0) {}
33
ComputeRelativeDelay(const Measurements & audio_measurement,const Measurements & video_measurement,int * relative_delay_ms)34 bool StreamSynchronization::ComputeRelativeDelay(
35 const Measurements& audio_measurement,
36 const Measurements& video_measurement,
37 int* relative_delay_ms) {
38 int64_t audio_last_capture_time_ms;
39 if (!audio_measurement.rtp_to_ntp.Estimate(audio_measurement.latest_timestamp,
40 &audio_last_capture_time_ms)) {
41 return false;
42 }
43 int64_t video_last_capture_time_ms;
44 if (!video_measurement.rtp_to_ntp.Estimate(video_measurement.latest_timestamp,
45 &video_last_capture_time_ms)) {
46 return false;
47 }
48 if (video_last_capture_time_ms < 0) {
49 return false;
50 }
51 // Positive diff means that video_measurement is behind audio_measurement.
52 *relative_delay_ms =
53 video_measurement.latest_receive_time_ms -
54 audio_measurement.latest_receive_time_ms -
55 (video_last_capture_time_ms - audio_last_capture_time_ms);
56
57 if (*relative_delay_ms > kMaxDeltaDelayMs ||
58 *relative_delay_ms < -kMaxDeltaDelayMs) {
59 return false;
60 }
61 return true;
62 }
63
ComputeDelays(int relative_delay_ms,int current_audio_delay_ms,int * total_audio_delay_target_ms,int * total_video_delay_target_ms)64 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
65 int current_audio_delay_ms,
66 int* total_audio_delay_target_ms,
67 int* total_video_delay_target_ms) {
68 int current_video_delay_ms = *total_video_delay_target_ms;
69
70 RTC_LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
71 << " current diff: " << relative_delay_ms
72 << " for stream " << audio_stream_id_;
73
74 // Calculate the difference between the lowest possible video delay and the
75 // current audio delay.
76 int current_diff_ms =
77 current_video_delay_ms - current_audio_delay_ms + relative_delay_ms;
78
79 avg_diff_ms_ =
80 ((kFilterLength - 1) * avg_diff_ms_ + current_diff_ms) / kFilterLength;
81 if (abs(avg_diff_ms_) < kMinDeltaMs) {
82 // Don't adjust if the diff is within our margin.
83 return false;
84 }
85
86 // Make sure we don't move too fast.
87 int diff_ms = avg_diff_ms_ / 2;
88 diff_ms = std::min(diff_ms, kMaxChangeMs);
89 diff_ms = std::max(diff_ms, -kMaxChangeMs);
90
91 // Reset the average after a move to prevent overshooting reaction.
92 avg_diff_ms_ = 0;
93
94 if (diff_ms > 0) {
95 // The minimum video delay is longer than the current audio delay.
96 // We need to decrease extra video delay, or add extra audio delay.
97 if (video_delay_.extra_ms > base_target_delay_ms_) {
98 // We have extra delay added to ViE. Reduce this delay before adding
99 // extra delay to VoE.
100 video_delay_.extra_ms -= diff_ms;
101 audio_delay_.extra_ms = base_target_delay_ms_;
102 } else { // video_delay_.extra_ms > 0
103 // We have no extra video delay to remove, increase the audio delay.
104 audio_delay_.extra_ms += diff_ms;
105 video_delay_.extra_ms = base_target_delay_ms_;
106 }
107 } else { // if (diff_ms > 0)
108 // The video delay is lower than the current audio delay.
109 // We need to decrease extra audio delay, or add extra video delay.
110 if (audio_delay_.extra_ms > base_target_delay_ms_) {
111 // We have extra delay in VoiceEngine.
112 // Start with decreasing the voice delay.
113 // Note: diff_ms is negative; add the negative difference.
114 audio_delay_.extra_ms += diff_ms;
115 video_delay_.extra_ms = base_target_delay_ms_;
116 } else { // audio_delay_.extra_ms > base_target_delay_ms_
117 // We have no extra delay in VoiceEngine, increase the video delay.
118 // Note: diff_ms is negative; subtract the negative difference.
119 video_delay_.extra_ms -= diff_ms; // X - (-Y) = X + Y.
120 audio_delay_.extra_ms = base_target_delay_ms_;
121 }
122 }
123
124 // Make sure that video is never below our target.
125 video_delay_.extra_ms =
126 std::max(video_delay_.extra_ms, base_target_delay_ms_);
127
128 int new_video_delay_ms;
129 if (video_delay_.extra_ms > base_target_delay_ms_) {
130 new_video_delay_ms = video_delay_.extra_ms;
131 } else {
132 // No change to the extra video delay. We are changing audio and we only
133 // allow to change one at the time.
134 new_video_delay_ms = video_delay_.last_ms;
135 }
136
137 // Make sure that we don't go below the extra video delay.
138 new_video_delay_ms = std::max(new_video_delay_ms, video_delay_.extra_ms);
139
140 // Verify we don't go above the maximum allowed video delay.
141 new_video_delay_ms =
142 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
143
144 int new_audio_delay_ms;
145 if (audio_delay_.extra_ms > base_target_delay_ms_) {
146 new_audio_delay_ms = audio_delay_.extra_ms;
147 } else {
148 // No change to the audio delay. We are changing video and we only allow to
149 // change one at the time.
150 new_audio_delay_ms = audio_delay_.last_ms;
151 }
152
153 // Make sure that we don't go below the extra audio delay.
154 new_audio_delay_ms = std::max(new_audio_delay_ms, audio_delay_.extra_ms);
155
156 // Verify we don't go above the maximum allowed audio delay.
157 new_audio_delay_ms =
158 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
159
160 video_delay_.last_ms = new_video_delay_ms;
161 audio_delay_.last_ms = new_audio_delay_ms;
162
163 RTC_LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
164 << " for video stream " << video_stream_id_
165 << " and audio delay " << audio_delay_.extra_ms
166 << " for audio stream " << audio_stream_id_;
167
168 *total_video_delay_target_ms = new_video_delay_ms;
169 *total_audio_delay_target_ms = new_audio_delay_ms;
170 return true;
171 }
172
SetTargetBufferingDelay(int target_delay_ms)173 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
174 // Initial extra delay for audio (accounting for existing extra delay).
175 audio_delay_.extra_ms += target_delay_ms - base_target_delay_ms_;
176 audio_delay_.last_ms += target_delay_ms - base_target_delay_ms_;
177
178 // The video delay is compared to the last value (and how much we can update
179 // is limited by that as well).
180 video_delay_.last_ms += target_delay_ms - base_target_delay_ms_;
181 video_delay_.extra_ms += target_delay_ms - base_target_delay_ms_;
182
183 // Video is already delayed by the desired amount.
184 base_target_delay_ms_ = target_delay_ms;
185 }
186
ReduceAudioDelay()187 void StreamSynchronization::ReduceAudioDelay() {
188 audio_delay_.extra_ms *= 0.9f;
189 }
190
ReduceVideoDelay()191 void StreamSynchronization::ReduceVideoDelay() {
192 video_delay_.extra_ms *= 0.9f;
193 }
194
195 } // namespace webrtc
196